1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22#include "llvm/IR/GlobalAlias.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsS390.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
41static cl::opt<bool> EnableIntArgExtCheck(
42 "argext-abi-check", cl::init(Val: false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
87static MachineOperand earlyUseOperand(MachineOperand Op) {
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
93SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::i64;
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
105 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
109 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
110 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
113 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
114 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
126 addRegisterClass(VT: MVT::v8f16, RC: &SystemZ::VR128BitRegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
128 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
133 }
134
135 // Compute derived properties from the register classes
136 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
137
138 // Set up special registers.
139 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
140
141 // TODO: It may be better to default to latency-oriented scheduling, however
142 // LLVM's current latency-oriented scheduler can't handle physreg definitions
143 // such as SystemZ has with CC, so set this to the register-pressure
144 // scheduler, because it can.
145 setSchedulingPreference(Sched::RegPressure);
146
147 setBooleanContents(ZeroOrOneBooleanContent);
148 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
149
150 setMaxAtomicSizeInBitsSupported(128);
151
152 // Instructions are strings of 2-byte aligned 2-byte values.
153 setMinFunctionAlignment(Align(2));
154 // For performance reasons we prefer 16-byte alignment.
155 setPrefFunctionAlignment(Align(16));
156
157 // Handle operations that are handled in a similar way for all types.
158 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
159 I <= MVT::LAST_FP_VALUETYPE;
160 ++I) {
161 MVT VT = MVT::SimpleValueType(I);
162 if (isTypeLegal(VT)) {
163 // Lower SET_CC into an IPM-based sequence.
164 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
165 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
166 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
167
168 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
169 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
170
171 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
172 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
173 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
174 }
175 }
176
177 // Expand jump table branches as address arithmetic followed by an
178 // indirect jump.
179 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
180
181 // Expand BRCOND into a BR_CC (see above).
182 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
183
184 // Handle integer types except i128.
185 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
186 I <= MVT::LAST_INTEGER_VALUETYPE;
187 ++I) {
188 MVT VT = MVT::SimpleValueType(I);
189 if (isTypeLegal(VT) && VT != MVT::i128) {
190 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
191
192 // Expand individual DIV and REMs into DIVREMs.
193 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
194 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
195 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
196 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
197 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
198 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
199
200 // Support addition/subtraction with overflow.
201 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
202 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
203
204 // Support addition/subtraction with carry.
205 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
206 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
207
208 // Support carry in as value rather than glue.
209 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
210 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
211
212 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
213 // available, or if the operand is constant.
214 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
215
216 // Use POPCNT on z196 and above.
217 if (Subtarget.hasPopulationCount())
218 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
219 else
220 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
221
222 // No special instructions for these.
223 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
224 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
225
226 // Use *MUL_LOHI where possible instead of MULH*.
227 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
228 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
229 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
230 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
231
232 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
233 // unsigned on z10 (only z196 and above have native support for
234 // unsigned conversions).
235 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
236 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
237 setOperationAction(Op, VT, Action: Custom);
238 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
239 setOperationAction(Op, VT, Action: Custom);
240 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
241 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
242 auto OpAction =
243 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
244 setOperationAction(Op, VT, Action: OpAction);
245 }
246 }
247 }
248
249 // Handle i128 if legal.
250 if (isTypeLegal(VT: MVT::i128)) {
251 // No special instructions for these.
252 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
253 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
254 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
255 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
256 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
257 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
258
259 // We may be able to use VSLDB/VSLD/VSRD for these.
260 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
261 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
262
263 // No special instructions for these before z17.
264 if (!Subtarget.hasVectorEnhancements3()) {
265 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
266 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
267 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
268 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
269 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
271 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
272 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
273 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
274 } else {
275 // Even if we do have a legal 128-bit multiply, we do not
276 // want 64-bit multiply-high operations to use it.
277 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
278 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
279 }
280
281 // Support addition/subtraction with carry.
282 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
283 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
284 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
285 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
286
287 // Use VPOPCT and add up partial results.
288 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
289
290 // Additional instructions available with z17.
291 if (Subtarget.hasVectorEnhancements3()) {
292 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
293
294 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
295 VT: MVT::i128, Action: Legal);
296 }
297 }
298
299 // These need custom handling in order to handle the f16 conversions.
300 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
301 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
302 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
303 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
304 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
305 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
306 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
307 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
308
309 // Type legalization will convert 8- and 16-bit atomic operations into
310 // forms that operate on i32s (but still keeping the original memory VT).
311 // Lower them into full i32 operations.
312 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: Custom);
313 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: Custom);
314 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: Custom);
315 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: Custom);
316 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: Custom);
317 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: Custom);
318 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: Custom);
319 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: Custom);
320 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: Custom);
321 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: Custom);
322 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: Custom);
323
324 // Whether or not i128 is not a legal type, we need to custom lower
325 // the atomic operations in order to exploit SystemZ instructions.
326 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
327 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
328 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
329 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
330
331 // Mark sign/zero extending atomic loads as legal, which will make
332 // DAGCombiner fold extensions into atomic loads if possible.
333 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
334 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
335 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
336 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
337 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
338 MemVT: MVT::i8, Action: Legal);
339
340 // We can use the CC result of compare-and-swap to implement
341 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
342 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
343 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
344 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
345
346 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
347
348 // Traps are legal, as we will convert them to "j .+2".
349 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
350
351 // We have native support for a 64-bit CTLZ, via FLOGR.
352 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
353 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
354 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
355
356 // On z17 we have native support for a 64-bit CTTZ.
357 if (Subtarget.hasMiscellaneousExtensions4()) {
358 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
359 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
360 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
361 }
362
363 // On z15 we have native support for a 64-bit CTPOP.
364 if (Subtarget.hasMiscellaneousExtensions3()) {
365 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
366 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
367 }
368
369 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
370 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
371
372 // Expand 128 bit shifts without using a libcall.
373 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
374 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
376
377 // Also expand 256 bit shifts if i128 is a legal type.
378 if (isTypeLegal(VT: MVT::i128)) {
379 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
380 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
381 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
382 }
383
384 // Handle bitcast from fp128 to i128.
385 if (!isTypeLegal(VT: MVT::i128))
386 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
387
388 // We have native instructions for i8, i16 and i32 extensions, but not i1.
389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
390 for (MVT VT : MVT::integer_valuetypes()) {
391 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
392 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
393 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
394 }
395
396 // Handle the various types of symbolic address.
397 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
398 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
399 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
400 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
401 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
402
403 // We need to handle dynamic allocations specially because of the
404 // 160-byte area at the bottom of the stack.
405 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
406 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
407
408 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
409 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
410
411 // Handle prefetches with PFD or PFDRL.
412 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
413
414 // Handle readcyclecounter with STCKF.
415 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
416
417 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
418 // Assume by default that all vector operations need to be expanded.
419 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
420 if (getOperationAction(Op: Opcode, VT) == Legal)
421 setOperationAction(Op: Opcode, VT, Action: Expand);
422
423 // Likewise all truncating stores and extending loads.
424 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
425 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
426 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
427 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
428 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
429 }
430
431 if (isTypeLegal(VT)) {
432 // These operations are legal for anything that can be stored in a
433 // vector register, even if there is no native support for the format
434 // as such. In particular, we can do these for v4f32 even though there
435 // are no specific instructions for that format.
436 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
437 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
438 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
439 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
440 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
441
442 // Likewise, except that we need to replace the nodes with something
443 // more specific.
444 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
445 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
446 }
447 }
448
449 // Handle integer vector types.
450 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
451 if (isTypeLegal(VT)) {
452 // These operations have direct equivalents.
453 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
454 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
455 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
456 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
457 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
458 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
459 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
460 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
461 }
462 if (Subtarget.hasVectorEnhancements3() &&
463 VT != MVT::v16i8 && VT != MVT::v8i16) {
464 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
465 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
466 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
467 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
468 }
469 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
470 setOperationAction(Op: ISD::AND, VT, Action: Legal);
471 setOperationAction(Op: ISD::OR, VT, Action: Legal);
472 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
473 if (Subtarget.hasVectorEnhancements1())
474 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
475 else
476 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
477 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
478 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
479
480 // Convert a GPR scalar to a vector by inserting it into element 0.
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
482
483 // Use a series of unpacks for extensions.
484 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
485 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
486
487 // Detect shifts/rotates by a scalar amount and convert them into
488 // V*_BY_SCALAR.
489 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
490 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
491 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
492 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
493
494 // Add ISD::VECREDUCE_ADD as custom in order to implement
495 // it with VZERO+VSUM
496 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
497
498 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
499 // and inverting the result as necessary.
500 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
501
502 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT,
503 Action: Legal);
504 }
505 }
506
507 if (Subtarget.hasVector()) {
508 // There should be no need to check for float types other than v2f64
509 // since <2 x f32> isn't a legal type.
510 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
511 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
512 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
513 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
514 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
515 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
516 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
517 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
518
519 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
520 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
521 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
522 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
523 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
524 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
525 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
526 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
527 }
528
529 if (Subtarget.hasVectorEnhancements2()) {
530 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
531 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
532 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
533 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
534 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
535 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
536 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
537 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
538
539 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
540 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
541 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
544 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
545 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
547 }
548
549 // Handle floating-point types.
550 if (!useSoftFloat()) {
551 // Promote all f16 operations to float, with some exceptions below.
552 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
553 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
554 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
555 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
556 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
557 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
558 }
559 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
560 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
561 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
562 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
563 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
564
565 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
566 setOperationAction(Op, VT: MVT::f16, Action: Legal);
567 }
568
569 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
570 I <= MVT::LAST_FP_VALUETYPE;
571 ++I) {
572 MVT VT = MVT::SimpleValueType(I);
573 if (isTypeLegal(VT) && VT != MVT::f16) {
574 // We can use FI for FRINT.
575 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
576
577 // We can use the extended form of FI for other rounding operations.
578 if (Subtarget.hasFPExtension()) {
579 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
580 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
581 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
582 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
583 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
584 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
585 }
586
587 // No special instructions for these.
588 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
589 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
590 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
591 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
592 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
593
594 // Special treatment.
595 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
596
597 // Handle constrained floating-point operations.
598 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
599 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
600 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
601 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
602 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
603 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
604 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
605 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
606 if (Subtarget.hasFPExtension()) {
607 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
608 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
609 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
610 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
611 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
612 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
613 }
614
615 // Extension from f16 needs libcall.
616 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
617 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
618 }
619 }
620
621 // Handle floating-point vector types.
622 if (Subtarget.hasVector()) {
623 // Scalar-to-vector conversion is just a subreg.
624 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8f16, Action: Legal);
625 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
626 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
627
628 // Some insertions and extractions can be done directly but others
629 // need to go via integers.
630 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
631 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
632 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
633 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
634 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
635 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
636
637 // These operations have direct equivalents.
638 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
639 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
640 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
641 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
642 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
643 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
644 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
645 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
646 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
653
654 // Handle constrained floating-point operations.
655 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
656 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
657 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
658 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
659 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
660 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
661 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
662 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
663 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
664 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
665 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
666 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
667 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
668
669 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
670 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
671 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
672 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
673 if (Subtarget.hasVectorEnhancements1()) {
674 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
675 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
676 }
677 }
678
679 // The vector enhancements facility 1 has instructions for these.
680 if (Subtarget.hasVectorEnhancements1()) {
681 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
682 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
683 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
684 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
685 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
686 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
687 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
688 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
689 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
690 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
691 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
692 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
693 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
694 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
695 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
696
697 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
698 setOperationAction(Op: ISD::FMAXNUM, VT: Type, Action: Legal);
699 setOperationAction(Op: ISD::FMAXIMUM, VT: Type, Action: Legal);
700 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: Type, Action: Legal);
701 setOperationAction(Op: ISD::PSEUDO_FMAX, VT: Type, Action: Legal);
702 setOperationAction(Op: ISD::FMINNUM, VT: Type, Action: Legal);
703 setOperationAction(Op: ISD::FMINIMUM, VT: Type, Action: Legal);
704 setOperationAction(Op: ISD::FMINIMUMNUM, VT: Type, Action: Legal);
705 setOperationAction(Op: ISD::PSEUDO_FMIN, VT: Type, Action: Legal);
706 }
707
708 // Handle constrained floating-point operations.
709 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
710 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
711 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
712 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
713 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
714 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
715 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
716 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
717 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
720 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
721 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
722 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
723 MVT::v4f32, MVT::v2f64 }) {
724 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
725 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
726 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
728 setOperationAction(Op: ISD::STRICT_PSEUDO_FMAX, VT, Action: Legal);
729 setOperationAction(Op: ISD::STRICT_PSEUDO_FMIN, VT, Action: Legal);
730 }
731 }
732
733 // We only have fused f128 multiply-addition on vector registers.
734 if (!Subtarget.hasVectorEnhancements1()) {
735 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
736 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
737 }
738
739 // We don't have a copysign instruction on vector registers.
740 if (Subtarget.hasVectorEnhancements1())
741 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
742
743 // Needed so that we don't try to implement f128 constant loads using
744 // a load-and-extend of a f80 constant (in cases where the constant
745 // would fit in an f80).
746 for (MVT VT : MVT::fp_valuetypes())
747 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
748
749 // We don't have extending load instruction on vector registers.
750 if (Subtarget.hasVectorEnhancements1()) {
751 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
752 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
753 }
754
755 // Floating-point truncation and stores need to be done separately.
756 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
757 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
758 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
759
760 // We have 64-bit FPR<->GPR moves, but need special handling for
761 // 32-bit forms.
762 if (!Subtarget.hasVector()) {
763 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
764 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
765 }
766
767 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
768 // structure, but VAEND is a no-op.
769 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
770 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
771 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
772
773 if (Subtarget.isTargetzOS()) {
774 // Handle address space casts between mixed sized pointers.
775 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
776 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
777 }
778
779 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
780
781 // Codes for which we want to perform some z-specific combinations.
782 setTargetDAGCombine({ISD::ZERO_EXTEND,
783 ISD::SIGN_EXTEND,
784 ISD::SIGN_EXTEND_INREG,
785 ISD::LOAD,
786 ISD::STORE,
787 ISD::VECTOR_SHUFFLE,
788 ISD::EXTRACT_VECTOR_ELT,
789 ISD::FP_ROUND,
790 ISD::STRICT_FP_ROUND,
791 ISD::FP_EXTEND,
792 ISD::SINT_TO_FP,
793 ISD::UINT_TO_FP,
794 ISD::STRICT_FP_EXTEND,
795 ISD::FCOPYSIGN,
796 ISD::BSWAP,
797 ISD::SETCC,
798 ISD::SRL,
799 ISD::SRA,
800 ISD::MUL,
801 ISD::SDIV,
802 ISD::UDIV,
803 ISD::SREM,
804 ISD::UREM,
805 ISD::INTRINSIC_VOID,
806 ISD::INTRINSIC_W_CHAIN});
807
808 // Handle intrinsics.
809 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
810 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
811
812 // We're not using SJLJ for exception handling, but they're implemented
813 // solely to support use of __builtin_setjmp / __builtin_longjmp.
814 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
815 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
816
817 // We want to use MVC in preference to even a single load/store pair.
818 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
819 MaxStoresPerMemcpyOptSize = 0;
820
821 // Same with memmove.
822 MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
823 MaxStoresPerMemmoveOptSize = 0;
824
825 // The main memset sequence is a byte store followed by an MVC.
826 // Two STC or MV..I stores win over that, but the kind of fused stores
827 // generated by target-independent code don't when the byte value is
828 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
829 // than "STC;MVC". Handle the choice in target-specific code instead.
830 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
831 MaxStoresPerMemsetOptSize = 0;
832
833 // Default to having -disable-strictnode-mutation on
834 IsStrictFPEnabled = true;
835}
836
837bool SystemZTargetLowering::useSoftFloat() const {
838 return Subtarget.hasSoftFloat();
839}
840
841unsigned SystemZTargetLowering::getVectorTypeBreakdownForCallingConv(
842 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
843 unsigned &NumIntermediates, MVT &RegisterVT) const {
844 // Pass fp16 vectors in VR(s).
845 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16)) {
846 IntermediateVT = RegisterVT = MVT::v8f16;
847 return NumIntermediates =
848 divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
849 }
850 return TargetLowering::getVectorTypeBreakdownForCallingConv(
851 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
852}
853
854MVT SystemZTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
855 CallingConv::ID CC,
856 EVT VT) const {
857 // 128-bit single-element vector types are passed like other vectors,
858 // not like their element type.
859 if (Subtarget.hasVector() && VT.isVector() && VT.getSizeInBits() == 128 &&
860 VT.getVectorNumElements() == 1)
861 return MVT::v16i8;
862 // Pass fp16 vectors in VR(s).
863 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
864 return MVT::v8f16;
865 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
866}
867
868unsigned SystemZTargetLowering::getNumRegistersForCallingConv(
869 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
870 // Pass fp16 vectors in VR(s).
871 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
872 return divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
873 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
874}
875
876EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
877 LLVMContext &, EVT VT) const {
878 if (!VT.isVector())
879 return MVT::i32;
880 return VT.changeVectorElementTypeToInteger();
881}
882
883bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
884 const MachineFunction &MF, EVT VT) const {
885 if (useSoftFloat())
886 return false;
887
888 VT = VT.getScalarType();
889
890 if (!VT.isSimple())
891 return false;
892
893 switch (VT.getSimpleVT().SimpleTy) {
894 case MVT::f32:
895 case MVT::f64:
896 return true;
897 case MVT::f128:
898 return Subtarget.hasVectorEnhancements1();
899 default:
900 break;
901 }
902
903 return false;
904}
905
906// Return true if the constant can be generated with a vector instruction,
907// such as VGM, VGMB or VREPI.
908bool SystemZVectorConstantInfo::isVectorConstantLegal(
909 const SystemZSubtarget &Subtarget) {
910 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
911 if (!Subtarget.hasVector() ||
912 (isFP128 && !Subtarget.hasVectorEnhancements1()))
913 return false;
914
915 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
916 // preferred way of creating all-zero and all-one vectors so give it
917 // priority over other methods below.
918 unsigned Mask = 0;
919 unsigned I = 0;
920 for (; I < SystemZ::VectorBytes; ++I) {
921 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
922 if (Byte == 0xff)
923 Mask |= 1ULL << I;
924 else if (Byte != 0)
925 break;
926 }
927 if (I == SystemZ::VectorBytes) {
928 Opcode = SystemZISD::BYTE_MASK;
929 OpVals.push_back(Elt: Mask);
930 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
931 return true;
932 }
933
934 if (SplatBitSize > 64)
935 return false;
936
937 auto TryValue = [&](uint64_t Value) -> bool {
938 // Try VECTOR REPLICATE IMMEDIATE
939 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
940 if (isInt<16>(x: SignedValue)) {
941 OpVals.push_back(Elt: ((unsigned) SignedValue));
942 Opcode = SystemZISD::REPLICATE;
943 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
944 NumElements: SystemZ::VectorBits / SplatBitSize);
945 return true;
946 }
947 // Try VECTOR GENERATE MASK
948 unsigned Start, End;
949 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
950 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
951 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
952 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
953 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
954 OpVals.push_back(Elt: End - (64 - SplatBitSize));
955 Opcode = SystemZISD::ROTATE_MASK;
956 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
957 NumElements: SystemZ::VectorBits / SplatBitSize);
958 return true;
959 }
960 return false;
961 };
962
963 // First try assuming that any undefined bits above the highest set bit
964 // and below the lowest set bit are 1s. This increases the likelihood of
965 // being able to use a sign-extended element value in VECTOR REPLICATE
966 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
967 uint64_t SplatBitsZ = SplatBits.getZExtValue();
968 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
969 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
970 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
971 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
972 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
973 if (TryValue(SplatBitsZ | Upper | Lower))
974 return true;
975
976 // Now try assuming that any undefined bits between the first and
977 // last defined set bits are set. This increases the chances of
978 // using a non-wraparound mask.
979 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
980 return TryValue(SplatBitsZ | Middle);
981}
982
983SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
984 if (IntImm.isSingleWord()) {
985 IntBits = APInt(128, IntImm.getZExtValue());
986 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
987 } else
988 IntBits = IntImm;
989 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
990
991 // Find the smallest splat.
992 SplatBits = IntImm;
993 unsigned Width = SplatBits.getBitWidth();
994 while (Width > 8) {
995 unsigned HalfSize = Width / 2;
996 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
997 APInt LowValue = SplatBits.trunc(width: HalfSize);
998
999 // If the two halves do not match, stop here.
1000 if (HighValue != LowValue || 8 > HalfSize)
1001 break;
1002
1003 SplatBits = HighValue;
1004 Width = HalfSize;
1005 }
1006 SplatUndef = 0;
1007 SplatBitSize = Width;
1008}
1009
1010SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
1011 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1012 bool HasAnyUndefs;
1013
1014 // Get IntBits by finding the 128 bit splat.
1015 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
1016 isBigEndian: true);
1017
1018 // Get SplatBits by finding the 8 bit or greater splat.
1019 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
1020 isBigEndian: true);
1021}
1022
1023bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1024 bool ForCodeSize) const {
1025 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1026 if (Imm.isZero() || Imm.isNegZero())
1027 return true;
1028
1029 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
1030}
1031
1032MachineBasicBlock *
1033SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
1034 MachineBasicBlock *MBB) const {
1035 DebugLoc DL = MI.getDebugLoc();
1036 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1037 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1038
1039 MachineFunction *MF = MBB->getParent();
1040 MachineRegisterInfo &MRI = MF->getRegInfo();
1041
1042 const BasicBlock *BB = MBB->getBasicBlock();
1043 MachineFunction::iterator I = ++MBB->getIterator();
1044
1045 Register DstReg = MI.getOperand(i: 0).getReg();
1046 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1047 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1048 (void)TRI;
1049 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1050 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1051
1052 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1053 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1054 // For v = setjmp(buf), we generate.
1055 // Algorithm:
1056 //
1057 // ---------
1058 // | thisMBB |
1059 // ---------
1060 // |
1061 // ------------------------
1062 // | |
1063 // ---------- ---------------
1064 // | mainMBB | | restoreMBB |
1065 // | v = 0 | | v = 1 |
1066 // ---------- ---------------
1067 // | |
1068 // -------------------------
1069 // |
1070 // -----------------------------
1071 // | sinkMBB |
1072 // | phi(v_mainMBB,v_restoreMBB) |
1073 // -----------------------------
1074 // thisMBB:
1075 // buf[FPOffset] = Frame Pointer if hasFP.
1076 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1077 // buf[BCOffset] = Backchain value if building with -mbackchain.
1078 // buf[SPOffset] = Stack Pointer.
1079 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1080 // SjLjSetup restoreMBB
1081 // mainMBB:
1082 // v_main = 0
1083 // sinkMBB:
1084 // v = phi(v_main, v_restore)
1085 // restoreMBB:
1086 // v_restore = 1
1087
1088 MachineBasicBlock *ThisMBB = MBB;
1089 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1090 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1091 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1092
1093 MF->insert(MBBI: I, MBB: MainMBB);
1094 MF->insert(MBBI: I, MBB: SinkMBB);
1095 MF->push_back(MBB: RestoreMBB);
1096 RestoreMBB->setMachineBlockAddressTaken();
1097
1098 MachineInstrBuilder MIB;
1099
1100 // Transfer the remainder of BB and its successor edges to sinkMBB.
1101 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1102 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1103 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1104
1105 // thisMBB:
1106 const int64_t FPOffset = 0; // Slot 1.
1107 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1108 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1109 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1110
1111 // Buf address.
1112 Register BufReg = MI.getOperand(i: 1).getReg();
1113
1114 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1115 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1116
1117 // Prepare IP for longjmp.
1118 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1119 .addMBB(MBB: RestoreMBB);
1120 // Store IP for return from jmp, slot 2, offset = 1.
1121 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1122 .addReg(RegNo: LabelReg)
1123 .addReg(RegNo: BufReg)
1124 .addImm(Val: LabelOffset)
1125 .addReg(RegNo: 0);
1126
1127 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1128 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1129 if (HasFP) {
1130 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1131 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1132 .addReg(RegNo: BufReg)
1133 .addImm(Val: FPOffset)
1134 .addReg(RegNo: 0);
1135 }
1136
1137 // Store SP.
1138 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1139 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1140 .addReg(RegNo: BufReg)
1141 .addImm(Val: SPOffset)
1142 .addReg(RegNo: 0);
1143
1144 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1145 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1146 if (BackChain) {
1147 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1148 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1149 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1150 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1151 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1152 .addReg(RegNo: 0);
1153
1154 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1155 .addReg(RegNo: BCReg)
1156 .addReg(RegNo: BufReg)
1157 .addImm(Val: BCOffset)
1158 .addReg(RegNo: 0);
1159 }
1160
1161 // Setup.
1162 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1163 .addMBB(MBB: RestoreMBB);
1164
1165 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1166 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1167
1168 ThisMBB->addSuccessor(Succ: MainMBB);
1169 ThisMBB->addSuccessor(Succ: RestoreMBB);
1170
1171 // mainMBB:
1172 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1173 MainMBB->addSuccessor(Succ: SinkMBB);
1174
1175 // sinkMBB:
1176 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1177 .addReg(RegNo: MainDstReg)
1178 .addMBB(MBB: MainMBB)
1179 .addReg(RegNo: RestoreDstReg)
1180 .addMBB(MBB: RestoreMBB);
1181
1182 // restoreMBB.
1183 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1184 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1185 RestoreMBB->addSuccessor(Succ: SinkMBB);
1186
1187 MI.eraseFromParent();
1188
1189 return SinkMBB;
1190}
1191
1192MachineBasicBlock *
1193SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1194 MachineBasicBlock *MBB) const {
1195
1196 DebugLoc DL = MI.getDebugLoc();
1197 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1198
1199 MachineFunction *MF = MBB->getParent();
1200 MachineRegisterInfo &MRI = MF->getRegInfo();
1201
1202 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1203 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1204 Register BufReg = MI.getOperand(i: 0).getReg();
1205 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1206 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1207
1208 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1209 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1210
1211 MachineInstrBuilder MIB;
1212
1213 const int64_t FPOffset = 0;
1214 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1215 const int64_t BCOffset = 2 * PVT.getStoreSize();
1216 const int64_t SPOffset = 3 * PVT.getStoreSize();
1217 const int64_t LPOffset = 4 * PVT.getStoreSize();
1218
1219 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1220 .addReg(RegNo: BufReg)
1221 .addImm(Val: LabelOffset)
1222 .addReg(RegNo: 0);
1223
1224 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1225 DestReg: SpecialRegs->getFramePointerRegister())
1226 .addReg(RegNo: BufReg)
1227 .addImm(Val: FPOffset)
1228 .addReg(RegNo: 0);
1229
1230 // We are restoring R13 even though we never stored in setjmp from llvm,
1231 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1232 // gcc setjmp and llvm longjmp.
1233 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1234 .addReg(RegNo: BufReg)
1235 .addImm(Val: LPOffset)
1236 .addReg(RegNo: 0);
1237
1238 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1239 if (BackChain) {
1240 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1241 .addReg(RegNo: BufReg)
1242 .addImm(Val: BCOffset)
1243 .addReg(RegNo: 0);
1244 }
1245
1246 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1247 DestReg: SpecialRegs->getStackPointerRegister())
1248 .addReg(RegNo: BufReg)
1249 .addImm(Val: SPOffset)
1250 .addReg(RegNo: 0);
1251
1252 if (BackChain) {
1253 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1254 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1255 .addReg(RegNo: BCReg)
1256 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1257 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1258 .addReg(RegNo: 0);
1259 }
1260
1261 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1262
1263 MI.eraseFromParent();
1264 return MBB;
1265}
1266
1267/// Returns true if stack probing through inline assembly is requested.
1268bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1269 // If the function specifically requests inline stack probes, emit them.
1270 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1271 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1272 "inline-asm";
1273 return false;
1274}
1275
1276TargetLowering::AtomicExpansionKind
1277SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1278 return AtomicExpansionKind::None;
1279}
1280
1281TargetLowering::AtomicExpansionKind
1282SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1283 return AtomicExpansionKind::None;
1284}
1285
1286TargetLowering::AtomicExpansionKind
1287SystemZTargetLowering::shouldExpandAtomicRMWInIR(
1288 const AtomicRMWInst *RMW) const {
1289 // Don't expand subword operations as they require special treatment.
1290 if (RMW->getType()->isIntegerTy(BitWidth: 8) || RMW->getType()->isIntegerTy(BitWidth: 16))
1291 return AtomicExpansionKind::None;
1292
1293 // Don't expand if there is a target instruction available.
1294 if (Subtarget.hasInterlockedAccess1() &&
1295 (RMW->getType()->isIntegerTy(BitWidth: 32) || RMW->getType()->isIntegerTy(BitWidth: 64)) &&
1296 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1297 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1298 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1299 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1300 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1301 return AtomicExpansionKind::None;
1302
1303 return AtomicExpansionKind::CmpXChg;
1304}
1305
1306bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1307 // We can use CGFI or CLGFI.
1308 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1309}
1310
1311bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1312 // We can use ALGFI or SLGFI.
1313 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1314}
1315
1316bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1317 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1318 // Unaligned accesses should never be slower than the expanded version.
1319 // We check specifically for aligned accesses in the few cases where
1320 // they are required.
1321 if (Fast)
1322 *Fast = 1;
1323 return true;
1324}
1325
1326bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1327 EVT VT = Y.getValueType();
1328
1329 // We can use NC(G)RK for types in GPRs ...
1330 if (VT == MVT::i32 || VT == MVT::i64)
1331 return Subtarget.hasMiscellaneousExtensions3();
1332
1333 // ... or VNC for types in VRs.
1334 if (VT.isVector() || VT == MVT::i128)
1335 return Subtarget.hasVector();
1336
1337 return false;
1338}
1339
1340// Information about the addressing mode for a memory access.
1341struct AddressingMode {
1342 // True if a long displacement is supported.
1343 bool LongDisplacement;
1344
1345 // True if use of index register is supported.
1346 bool IndexReg;
1347
1348 AddressingMode(bool LongDispl, bool IdxReg) :
1349 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1350};
1351
1352// Return the desired addressing mode for a Load which has only one use (in
1353// the same block) which is a Store.
1354static AddressingMode getLoadStoreAddrMode(bool HasVector,
1355 Type *Ty) {
1356 // With vector support a Load->Store combination may be combined to either
1357 // an MVC or vector operations and it seems to work best to allow the
1358 // vector addressing mode.
1359 if (HasVector)
1360 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1361
1362 // Otherwise only the MVC case is special.
1363 bool MVC = Ty->isIntegerTy(BitWidth: 8);
1364 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1365}
1366
1367// Return the addressing mode which seems most desirable given an LLVM
1368// Instruction pointer.
1369static AddressingMode
1370supportedAddressingMode(Instruction *I, bool HasVector) {
1371 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1372 switch (II->getIntrinsicID()) {
1373 default: break;
1374 case Intrinsic::memset:
1375 case Intrinsic::memmove:
1376 case Intrinsic::memcpy:
1377 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1378 }
1379 }
1380
1381 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1382 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1383 if (SingleUser->getParent() == I->getParent()) {
1384 if (isa<ICmpInst>(Val: SingleUser)) {
1385 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1386 if (C->getBitWidth() <= 64 &&
1387 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1388 // Comparison of memory with 16 bit signed / unsigned immediate
1389 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1390 } else if (isa<StoreInst>(Val: SingleUser))
1391 // Load->Store
1392 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1393 }
1394 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1395 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1396 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1397 // Load->Store
1398 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1399 }
1400
1401 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1402
1403 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1404 // dependencies (LDE only supports small offsets).
1405 // * Utilize the vector registers to hold floating point
1406 // values (vector load / store instructions only support small
1407 // offsets).
1408
1409 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1410 I->getOperand(i: 0)->getType());
1411 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1412 bool IsVectorAccess = MemAccessTy->isVectorTy();
1413
1414 // A store of an extracted vector element will be combined into a VSTE type
1415 // instruction.
1416 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1417 Value *DataOp = I->getOperand(i: 0);
1418 if (isa<ExtractElementInst>(Val: DataOp))
1419 IsVectorAccess = true;
1420 }
1421
1422 // A load which gets inserted into a vector element will be combined into a
1423 // VLE type instruction.
1424 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1425 User *LoadUser = *I->user_begin();
1426 if (isa<InsertElementInst>(Val: LoadUser))
1427 IsVectorAccess = true;
1428 }
1429
1430 if (IsFPAccess || IsVectorAccess)
1431 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1432 }
1433
1434 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1435}
1436
1437bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1438 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1439 // Punt on globals for now, although they can be used in limited
1440 // RELATIVE LONG cases.
1441 if (AM.BaseGV)
1442 return false;
1443
1444 // Require a 20-bit signed offset.
1445 if (!isInt<20>(x: AM.BaseOffs))
1446 return false;
1447
1448 bool RequireD12 =
1449 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(BitWidth: 128));
1450 AddressingMode SupportedAM(!RequireD12, true);
1451 if (I != nullptr)
1452 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1453
1454 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1455 return false;
1456
1457 if (!SupportedAM.IndexReg)
1458 // No indexing allowed.
1459 return AM.Scale == 0;
1460 else
1461 // Indexing is OK but no scale factor can be applied.
1462 return AM.Scale == 0 || AM.Scale == 1;
1463}
1464
1465bool SystemZTargetLowering::findOptimalMemOpLowering(
1466 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1467 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1468 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1469
1470 assert(Limit != ~0U &&
1471 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1472
1473 if (Op.isZeroMemset())
1474 return false; // Memset zero: Use XC.
1475
1476 const int MVCFastLen = 16;
1477 // Use MVC up to 16 bytes for memcpy. Small memset uses STC/MVI for first
1478 // byte.
1479 if (Op.isMemcpy() && Op.size() <= MVCFastLen)
1480 return false;
1481 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1482 return false;
1483
1484 // Avoid unaligned VL/VST:s.
1485 if ((Op.size() >= 16 && !Op.isAligned(AlignCheck: Align(8))) ||
1486 (Op.size() >= 25 && Op.size() <= 31))
1487 return false;
1488
1489 return TargetLowering::findOptimalMemOpLowering(
1490 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1491}
1492
1493EVT SystemZTargetLowering::getOptimalMemOpType(
1494 LLVMContext &Context, const MemOp &Op,
1495 const AttributeList &FuncAttributes) const {
1496 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1497}
1498
1499bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1500 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1501 return false;
1502 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1503 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1504 return FromBits > ToBits;
1505}
1506
1507bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1508 if (!FromVT.isInteger() || !ToVT.isInteger())
1509 return false;
1510 unsigned FromBits = FromVT.getFixedSizeInBits();
1511 unsigned ToBits = ToVT.getFixedSizeInBits();
1512 return FromBits > ToBits;
1513}
1514
1515//===----------------------------------------------------------------------===//
1516// Inline asm support
1517//===----------------------------------------------------------------------===//
1518
1519TargetLowering::ConstraintType
1520SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1521 if (Constraint.size() == 1) {
1522 switch (Constraint[0]) {
1523 case 'a': // Address register
1524 case 'd': // Data register (equivalent to 'r')
1525 case 'f': // Floating-point register
1526 case 'h': // High-part register
1527 case 'r': // General-purpose register
1528 case 'v': // Vector register
1529 return C_RegisterClass;
1530
1531 case 'Q': // Memory with base and unsigned 12-bit displacement
1532 case 'R': // Likewise, plus an index
1533 case 'S': // Memory with base and signed 20-bit displacement
1534 case 'T': // Likewise, plus an index
1535 case 'm': // Equivalent to 'T'.
1536 return C_Memory;
1537
1538 case 'I': // Unsigned 8-bit constant
1539 case 'J': // Unsigned 12-bit constant
1540 case 'K': // Signed 16-bit constant
1541 case 'L': // Signed 20-bit displacement (on all targets we support)
1542 case 'M': // 0x7fffffff
1543 return C_Immediate;
1544
1545 default:
1546 break;
1547 }
1548 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1549 switch (Constraint[1]) {
1550 case 'Q': // Address with base and unsigned 12-bit displacement
1551 case 'R': // Likewise, plus an index
1552 case 'S': // Address with base and signed 20-bit displacement
1553 case 'T': // Likewise, plus an index
1554 return C_Address;
1555
1556 default:
1557 break;
1558 }
1559 } else if (Constraint.size() == 5 && Constraint.starts_with(Prefix: "{")) {
1560 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1561 return C_Other;
1562 }
1563 return TargetLowering::getConstraintType(Constraint);
1564}
1565
1566TargetLowering::ConstraintWeight
1567SystemZTargetLowering::getSingleConstraintMatchWeight(
1568 AsmOperandInfo &Info, const char *Constraint) const {
1569 ConstraintWeight Weight = CW_Invalid;
1570 Value *CallOperandVal = Info.CallOperandVal;
1571 // If we don't have a value, we can't do a match,
1572 // but allow it at the lowest weight.
1573 if (!CallOperandVal)
1574 return CW_Default;
1575 Type *type = CallOperandVal->getType();
1576 // Look at the constraint type.
1577 switch (*Constraint) {
1578 default:
1579 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1580 break;
1581
1582 case 'a': // Address register
1583 case 'd': // Data register (equivalent to 'r')
1584 case 'h': // High-part register
1585 case 'r': // General-purpose register
1586 Weight =
1587 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1588 break;
1589
1590 case 'f': // Floating-point register
1591 if (!useSoftFloat())
1592 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1593 break;
1594
1595 case 'v': // Vector register
1596 if (Subtarget.hasVector())
1597 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1598 : CW_Default;
1599 break;
1600
1601 case 'I': // Unsigned 8-bit constant
1602 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1603 if (isUInt<8>(x: C->getZExtValue()))
1604 Weight = CW_Constant;
1605 break;
1606
1607 case 'J': // Unsigned 12-bit constant
1608 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1609 if (isUInt<12>(x: C->getZExtValue()))
1610 Weight = CW_Constant;
1611 break;
1612
1613 case 'K': // Signed 16-bit constant
1614 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1615 if (isInt<16>(x: C->getSExtValue()))
1616 Weight = CW_Constant;
1617 break;
1618
1619 case 'L': // Signed 20-bit displacement (on all targets we support)
1620 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1621 if (isInt<20>(x: C->getSExtValue()))
1622 Weight = CW_Constant;
1623 break;
1624
1625 case 'M': // 0x7fffffff
1626 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1627 if (C->getZExtValue() == 0x7fffffff)
1628 Weight = CW_Constant;
1629 break;
1630 }
1631 return Weight;
1632}
1633
1634// Parse a "{tNNN}" register constraint for which the register type "t"
1635// has already been verified. MC is the class associated with "t" and
1636// Map maps 0-based register numbers to LLVM register numbers.
1637static std::pair<unsigned, const TargetRegisterClass *>
1638parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1639 const unsigned *Map, unsigned Size) {
1640 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1641 if (isdigit(Constraint[2])) {
1642 unsigned Index;
1643 bool Failed =
1644 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1645 if (!Failed && Index < Size && Map[Index])
1646 return std::make_pair(x: Map[Index], y&: RC);
1647 }
1648 return std::make_pair(x: 0U, y: nullptr);
1649}
1650
1651std::pair<unsigned, const TargetRegisterClass *>
1652SystemZTargetLowering::getRegForInlineAsmConstraint(
1653 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1654 if (Constraint.size() == 1) {
1655 // GCC Constraint Letters
1656 switch (Constraint[0]) {
1657 default: break;
1658 case 'd': // Data register (equivalent to 'r')
1659 case 'r': // General-purpose register
1660 if (VT.getSizeInBits() == 64)
1661 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1662 else if (VT.getSizeInBits() == 128)
1663 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1664 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1665
1666 case 'a': // Address register
1667 if (VT == MVT::i64)
1668 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1669 else if (VT == MVT::i128)
1670 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1671 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1672
1673 case 'h': // High-part register (an LLVM extension)
1674 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1675
1676 case 'f': // Floating-point register
1677 if (!useSoftFloat()) {
1678 if (VT.getSizeInBits() == 16)
1679 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1680 else if (VT.getSizeInBits() == 64)
1681 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1682 else if (VT.getSizeInBits() == 128)
1683 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1684 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1685 }
1686 break;
1687
1688 case 'v': // Vector register
1689 if (Subtarget.hasVector()) {
1690 if (VT.getSizeInBits() == 16)
1691 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1692 if (VT.getSizeInBits() == 32)
1693 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1694 if (VT.getSizeInBits() == 64)
1695 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1696 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1697 }
1698 break;
1699 }
1700 }
1701 if (Constraint.starts_with(Prefix: "{")) {
1702
1703 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1704 // to check the size on.
1705 auto getVTSizeInBits = [&VT]() {
1706 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1707 };
1708
1709 // We need to override the default register parsing for GPRs and FPRs
1710 // because the interpretation depends on VT. The internal names of
1711 // the registers are also different from the external names
1712 // (F0D and F0S instead of F0, etc.).
1713 if (Constraint[1] == 'r') {
1714 if (getVTSizeInBits() == 32)
1715 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1716 Map: SystemZMC::GR32Regs, Size: 16);
1717 if (getVTSizeInBits() == 128)
1718 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1719 Map: SystemZMC::GR128Regs, Size: 16);
1720 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1721 Map: SystemZMC::GR64Regs, Size: 16);
1722 }
1723 if (Constraint[1] == 'f') {
1724 if (useSoftFloat())
1725 return std::make_pair(
1726 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1727 if (getVTSizeInBits() == 16)
1728 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1729 Map: SystemZMC::FP16Regs, Size: 16);
1730 if (getVTSizeInBits() == 32)
1731 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1732 Map: SystemZMC::FP32Regs, Size: 16);
1733 if (getVTSizeInBits() == 128)
1734 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1735 Map: SystemZMC::FP128Regs, Size: 16);
1736 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1737 Map: SystemZMC::FP64Regs, Size: 16);
1738 }
1739 if (Constraint[1] == 'v') {
1740 if (!Subtarget.hasVector())
1741 return std::make_pair(
1742 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1743 if (getVTSizeInBits() == 16)
1744 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1745 Map: SystemZMC::VR16Regs, Size: 32);
1746 if (getVTSizeInBits() == 32)
1747 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1748 Map: SystemZMC::VR32Regs, Size: 32);
1749 if (getVTSizeInBits() == 64)
1750 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1751 Map: SystemZMC::VR64Regs, Size: 32);
1752 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1753 Map: SystemZMC::VR128Regs, Size: 32);
1754 }
1755 if (Constraint[1] == '@') {
1756 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1757 return std::make_pair(x: SystemZ::CC, y: &SystemZ::CCRRegClass);
1758 }
1759 }
1760 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1761}
1762
1763// FIXME? Maybe this could be a TableGen attribute on some registers and
1764// this table could be generated automatically from RegInfo.
1765Register
1766SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1767 const MachineFunction &MF) const {
1768 Register Reg =
1769 StringSwitch<Register>(RegName)
1770 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1771 : SystemZ::NoRegister)
1772 .Case(S: "r15",
1773 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1774 .Default(Value: Register());
1775
1776 return Reg;
1777}
1778
1779Register SystemZTargetLowering::getExceptionPointerRegister(
1780 ExceptionHandling EH, const Constant *PersonalityFn) const {
1781 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1782}
1783
1784Register SystemZTargetLowering::getExceptionSelectorRegister(
1785 ExceptionHandling EH, const Constant *PersonalityFn) const {
1786 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1787}
1788
1789// Convert condition code in CCReg to an i32 value.
1790static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
1791 SDLoc DL(CCReg);
1792 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
1793 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
1794 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
1795}
1796
1797// Lower @cc targets via setcc.
1798SDValue SystemZTargetLowering::LowerAsmOutputForConstraint(
1799 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1800 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1801 if (StringRef("{@cc}").compare(RHS: OpInfo.ConstraintCode) != 0)
1802 return SDValue();
1803
1804 // Check that return type is valid.
1805 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1806 OpInfo.ConstraintVT.getSizeInBits() < 8)
1807 report_fatal_error(reason: "Glue output operand is of invalid type");
1808
1809 if (Glue.getNode()) {
1810 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32, Glue);
1811 Chain = Glue.getValue(R: 1);
1812 } else
1813 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32);
1814 return getCCResult(DAG, CCReg: Glue);
1815}
1816
1817void SystemZTargetLowering::LowerAsmOperandForConstraint(
1818 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1819 SelectionDAG &DAG) const {
1820 // Only support length 1 constraints for now.
1821 if (Constraint.size() == 1) {
1822 switch (Constraint[0]) {
1823 case 'I': // Unsigned 8-bit constant
1824 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1825 if (isUInt<8>(x: C->getZExtValue()))
1826 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1827 VT: Op.getValueType()));
1828 return;
1829
1830 case 'J': // Unsigned 12-bit constant
1831 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1832 if (isUInt<12>(x: C->getZExtValue()))
1833 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1834 VT: Op.getValueType()));
1835 return;
1836
1837 case 'K': // Signed 16-bit constant
1838 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1839 if (isInt<16>(x: C->getSExtValue()))
1840 Ops.push_back(x: DAG.getSignedTargetConstant(
1841 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1842 return;
1843
1844 case 'L': // Signed 20-bit displacement (on all targets we support)
1845 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1846 if (isInt<20>(x: C->getSExtValue()))
1847 Ops.push_back(x: DAG.getSignedTargetConstant(
1848 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1849 return;
1850
1851 case 'M': // 0x7fffffff
1852 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1853 if (C->getZExtValue() == 0x7fffffff)
1854 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1855 VT: Op.getValueType()));
1856 return;
1857 }
1858 }
1859 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1860}
1861
1862//===----------------------------------------------------------------------===//
1863// Calling conventions
1864//===----------------------------------------------------------------------===//
1865
1866#define GET_CALLING_CONV_IMPL
1867#include "SystemZGenCallingConv.inc"
1868
1869const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1870 CallingConv::ID) const {
1871 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1872 SystemZ::R14D, 0 };
1873 return ScratchRegs;
1874}
1875
1876bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1877 Type *ToType) const {
1878 return isTruncateFree(FromType, ToType);
1879}
1880
1881bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1882 return CI->isTailCall();
1883}
1884
1885// Value is a value that has been passed to us in the location described by VA
1886// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1887// any loads onto Chain.
1888static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1889 CCValAssign &VA, SDValue Chain,
1890 SDValue Value) {
1891 // If the argument has been promoted from a smaller type, insert an
1892 // assertion to capture this.
1893 if (VA.getLocInfo() == CCValAssign::SExt)
1894 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1895 N2: DAG.getValueType(VA.getValVT()));
1896 else if (VA.getLocInfo() == CCValAssign::ZExt)
1897 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1898 N2: DAG.getValueType(VA.getValVT()));
1899
1900 if (VA.isExtInLoc())
1901 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1902 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1903 // If this is a short vector argument loaded from the stack,
1904 // extend from i64 to full vector size and then bitcast.
1905 assert(VA.getLocVT() == MVT::i64);
1906 assert(VA.getValVT().isVector());
1907 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1908 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1909 } else
1910 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1911 return Value;
1912}
1913
1914// Value is a value of type VA.getValVT() that we need to copy into
1915// the location described by VA. Return a copy of Value converted to
1916// VA.getValVT(). The caller is responsible for handling indirect values.
1917static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1918 CCValAssign &VA, SDValue Value) {
1919 switch (VA.getLocInfo()) {
1920 case CCValAssign::SExt:
1921 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1922 case CCValAssign::ZExt:
1923 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1924 case CCValAssign::AExt:
1925 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1926 case CCValAssign::BCvt: {
1927 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1928 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1929 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1930 // For an f32 vararg we need to first promote it to an f64 and then
1931 // bitcast it to an i64.
1932 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1933 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
1934 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1935 ? MVT::v2i64
1936 : VA.getLocVT();
1937 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
1938 // For ELF, this is a short vector argument to be stored to the stack,
1939 // bitcast to v2i64 and then extract first element.
1940 if (BitCastToType == MVT::v2i64)
1941 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
1942 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1943 return Value;
1944 }
1945 case CCValAssign::Full:
1946 return Value;
1947 default:
1948 llvm_unreachable("Unhandled getLocInfo()");
1949 }
1950}
1951
1952static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1953 SDLoc DL(In);
1954 SDValue Lo, Hi;
1955 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1956 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
1957 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
1958 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
1959 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
1960 } else {
1961 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
1962 }
1963
1964 // FIXME: If v2i64 were a legal type, we could use it instead of
1965 // Untyped here. This might enable improved folding.
1966 SDNode *Pair = DAG.getMachineNode(Opcode: SystemZ::PAIR128, dl: DL,
1967 VT: MVT::Untyped, Op1: Hi, Op2: Lo);
1968 return SDValue(Pair, 0);
1969}
1970
1971static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1972 SDLoc DL(In);
1973 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
1974 DL, VT: MVT::i64, Operand: In);
1975 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
1976 DL, VT: MVT::i64, Operand: In);
1977
1978 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1979 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
1980 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
1981 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
1982 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
1983 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1984 } else {
1985 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1986 }
1987}
1988
1989bool SystemZTargetLowering::splitValueIntoRegisterParts(
1990 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1991 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
1992 EVT ValueVT = Val.getValueType();
1993 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1994 // Inline assembly operand.
1995 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
1996 return true;
1997 }
1998
1999 return false;
2000}
2001
2002SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
2003 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
2004 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
2005 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2006 // Inline assembly operand.
2007 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
2008 return DAG.getBitcast(VT: ValueVT, V: Res);
2009 }
2010
2011 return SDValue();
2012}
2013
2014// The first part of a split stack argument is at index I in Args (and
2015// ArgLocs). Return the type of a part and the number of them by reference.
2016template <class ArgTy>
2017static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args,
2018 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2019 MVT &PartVT, unsigned &NumParts) {
2020 if (!Args[I].Flags.isSplit())
2021 return false;
2022 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2023 "ArgLocs havoc.");
2024 PartVT = ArgLocs[I].getValVT();
2025 NumParts = 1;
2026 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2027 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2028 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2029 ++NumParts;
2030 if (Args[PartIdx].Flags.isSplitEnd())
2031 break;
2032 }
2033 return true;
2034}
2035
2036SDValue SystemZTargetLowering::LowerFormalArguments(
2037 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2038 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2039 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2040 MachineFunction &MF = DAG.getMachineFunction();
2041 MachineFrameInfo &MFI = MF.getFrameInfo();
2042 MachineRegisterInfo &MRI = MF.getRegInfo();
2043 SystemZMachineFunctionInfo *FuncInfo =
2044 MF.getInfo<SystemZMachineFunctionInfo>();
2045 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2046 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2047
2048 // Assign locations to all of the incoming arguments.
2049 SmallVector<CCValAssign, 16> ArgLocs;
2050 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2051 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
2052 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2053
2054 unsigned NumFixedGPRs = 0;
2055 unsigned NumFixedFPRs = 0;
2056 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2057 SDValue ArgValue;
2058 CCValAssign &VA = ArgLocs[I];
2059 EVT LocVT = VA.getLocVT();
2060 if (VA.isRegLoc()) {
2061 // Arguments passed in registers
2062 const TargetRegisterClass *RC;
2063 switch (LocVT.getSimpleVT().SimpleTy) {
2064 default:
2065 // Integers smaller than i64 should be promoted to i64.
2066 llvm_unreachable("Unexpected argument type");
2067 case MVT::i32:
2068 NumFixedGPRs += 1;
2069 RC = &SystemZ::GR32BitRegClass;
2070 break;
2071 case MVT::i64:
2072 NumFixedGPRs += 1;
2073 RC = &SystemZ::GR64BitRegClass;
2074 break;
2075 case MVT::f16:
2076 NumFixedFPRs += 1;
2077 RC = &SystemZ::FP16BitRegClass;
2078 break;
2079 case MVT::f32:
2080 NumFixedFPRs += 1;
2081 RC = &SystemZ::FP32BitRegClass;
2082 break;
2083 case MVT::f64:
2084 NumFixedFPRs += 1;
2085 RC = &SystemZ::FP64BitRegClass;
2086 break;
2087 case MVT::f128:
2088 NumFixedFPRs += 2;
2089 RC = &SystemZ::FP128BitRegClass;
2090 break;
2091 case MVT::v16i8:
2092 case MVT::v8i16:
2093 case MVT::v4i32:
2094 case MVT::v2i64:
2095 case MVT::v8f16:
2096 case MVT::v4f32:
2097 case MVT::v2f64:
2098 RC = &SystemZ::VR128BitRegClass;
2099 break;
2100 }
2101
2102 Register VReg = MRI.createVirtualRegister(RegClass: RC);
2103 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
2104 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
2105 } else {
2106 assert(VA.isMemLoc() && "Argument not register or memory");
2107
2108 // Create the frame index object for this incoming parameter.
2109 // FIXME: Pre-include call frame size in the offset, should not
2110 // need to manually add it here.
2111 int64_t ArgSPOffset = VA.getLocMemOffset();
2112 if (Subtarget.isTargetXPLINK64()) {
2113 auto &XPRegs =
2114 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2115 ArgSPOffset += XPRegs.getCallFrameSize();
2116 }
2117 int FI =
2118 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2119
2120 // Create the SelectionDAG nodes corresponding to a load
2121 // from this parameter. Unpromoted ints and floats are
2122 // passed as right-justified 8-byte values.
2123 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2124 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2125 VA.getLocVT() == MVT::f16) {
2126 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2127 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2128 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2129 }
2130 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2131 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2132 }
2133
2134 // Convert the value of the argument register into the value that's
2135 // being passed.
2136 if (VA.getLocInfo() == CCValAssign::Indirect) {
2137 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2138 PtrInfo: MachinePointerInfo()));
2139 // If the original argument was split (e.g. i128), we need
2140 // to load all parts of it here (using the same address).
2141 MVT PartVT;
2142 unsigned NumParts;
2143 if (analyzeArgSplit(Args: Ins, ArgLocs, I, PartVT, NumParts)) {
2144 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2145 ++I;
2146 CCValAssign &PartVA = ArgLocs[I];
2147 unsigned PartOffset = Ins[I].PartOffset;
2148 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2149 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2150 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2151 PtrInfo: MachinePointerInfo()));
2152 assert(PartOffset && "Offset should be non-zero.");
2153 }
2154 }
2155 } else if (Subtarget.isTargetXPLINK64() &&
2156 (VA.getLocInfo() == CCValAssign::SExt ||
2157 VA.getLocInfo() == CCValAssign::ZExt) &&
2158 Ins[I].ArgVT.isSimple()) {
2159 // Some prior z/OS compilers do not always perform the extension of
2160 // short integer arguments or pointers. To accommodate those, do not
2161 // rely on that extension by avoiding any AssertSext/AssertZext nodes by
2162 // directly truncating ArgValue to the original argument type.
2163 MVT OrigVT = Ins[I].ArgVT.getSimpleVT();
2164 InVals.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: ArgValue));
2165 } else
2166 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2167 }
2168
2169 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2170 // Save the number of non-varargs registers for later use by va_start, etc.
2171 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2172 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2173
2174 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2175 Subtarget.getSpecialRegisters());
2176
2177 // Likewise the address (in the form of a frame index) of where the
2178 // first stack vararg would be. The 1-byte size here is arbitrary.
2179 // FIXME: Pre-include call frame size in the offset, should not
2180 // need to manually add it here.
2181 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2182 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2183 FuncInfo->setVarArgsFrameIndex(FI);
2184 }
2185
2186 if (IsVarArg && Subtarget.isTargetELF()) {
2187 // Save the number of non-varargs registers for later use by va_start, etc.
2188 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2189 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2190
2191 // Likewise the address (in the form of a frame index) of where the
2192 // first stack vararg would be. The 1-byte size here is arbitrary.
2193 int64_t VarArgsOffset = CCInfo.getStackSize();
2194 FuncInfo->setVarArgsFrameIndex(
2195 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2196
2197 // ...and a similar frame index for the caller-allocated save area
2198 // that will be used to store the incoming registers.
2199 int64_t RegSaveOffset =
2200 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2201 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2202 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2203
2204 // Store the FPR varargs in the reserved frame slots. (We store the
2205 // GPRs as part of the prologue.)
2206 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2207 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2208 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2209 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2210 int FI =
2211 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2212 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2213 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2214 RC: &SystemZ::FP64BitRegClass);
2215 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2216 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2217 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2218 }
2219 // Join the stores, which are independent of one another.
2220 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2221 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2222 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2223 }
2224 }
2225
2226 if (Subtarget.isTargetXPLINK64()) {
2227 // Create virual register for handling incoming "ADA" special register (R5)
2228 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
2229 Register ADAvReg = MRI.createVirtualRegister(RegClass: RC);
2230 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2231 Subtarget.getSpecialRegisters());
2232 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2233 FuncInfo->setADAVirtualRegister(ADAvReg);
2234 }
2235 return Chain;
2236}
2237
2238static bool canUseSiblingCall(const CCState &ArgCCInfo,
2239 SmallVectorImpl<CCValAssign> &ArgLocs,
2240 SmallVectorImpl<ISD::OutputArg> &Outs) {
2241 // Punt if there are any indirect or stack arguments, or if the call
2242 // needs the callee-saved argument register R6, or if the call uses
2243 // the callee-saved register arguments SwiftSelf and SwiftError.
2244 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2245 CCValAssign &VA = ArgLocs[I];
2246 if (VA.getLocInfo() == CCValAssign::Indirect)
2247 return false;
2248 if (!VA.isRegLoc())
2249 return false;
2250 Register Reg = VA.getLocReg();
2251 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2252 return false;
2253 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2254 return false;
2255 }
2256 return true;
2257}
2258
2259static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2260 unsigned Offset, bool LoadAdr = false) {
2261 MachineFunction &MF = DAG.getMachineFunction();
2262 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
2263 Register ADAvReg = MFI->getADAVirtualRegister();
2264 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2265
2266 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2267 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2268
2269 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2270 if (!LoadAdr)
2271 Result = DAG.getLoad(
2272 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2273 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2274
2275 return Result;
2276}
2277
2278// ADA access using Global value
2279// Note: for functions, address of descriptor is returned
2280static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2281 EVT PtrVT) {
2282 unsigned ADAtype;
2283 bool LoadAddr = false;
2284 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2285 bool IsFunction =
2286 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2287 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2288
2289 if (IsFunction) {
2290 if (IsInternal) {
2291 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2292 LoadAddr = true;
2293 } else
2294 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2295 } else {
2296 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2297 }
2298 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2299
2300 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2301}
2302
2303static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2304 SDLoc &DL, SDValue &Chain) {
2305 unsigned ADADelta = 0; // ADA offset in desc.
2306 unsigned EPADelta = 8; // EPA offset in desc.
2307 MachineFunction &MF = DAG.getMachineFunction();
2308 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2309
2310 // XPLink calling convention.
2311 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2312 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2313 G->getGlobal()->hasPrivateLinkage());
2314 if (IsInternal) {
2315 SystemZMachineFunctionInfo *MFI =
2316 MF.getInfo<SystemZMachineFunctionInfo>();
2317 Register ADAvReg = MFI->getADAVirtualRegister();
2318 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2319 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2320 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2321 return true;
2322 } else {
2323 SDValue GA = DAG.getTargetGlobalAddress(
2324 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2325 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2326 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2327 }
2328 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2329 SDValue ES = DAG.getTargetExternalSymbol(
2330 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2331 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2332 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2333 } else {
2334 // Function pointer case
2335 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2336 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2337 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2338 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2339 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2340 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2341 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2342 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2343 }
2344 return false;
2345}
2346
2347SDValue
2348SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2349 SmallVectorImpl<SDValue> &InVals) const {
2350 SelectionDAG &DAG = CLI.DAG;
2351 SDLoc &DL = CLI.DL;
2352 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2353 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2354 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2355 SDValue Chain = CLI.Chain;
2356 SDValue Callee = CLI.Callee;
2357 bool &IsTailCall = CLI.IsTailCall;
2358 CallingConv::ID CallConv = CLI.CallConv;
2359 bool IsVarArg = CLI.IsVarArg;
2360 MachineFunction &MF = DAG.getMachineFunction();
2361 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2362 LLVMContext &Ctx = *DAG.getContext();
2363 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2364
2365 // FIXME: z/OS support to be added in later.
2366 if (Subtarget.isTargetXPLINK64())
2367 IsTailCall = false;
2368
2369 // Integer args <=32 bits should have an extension attribute.
2370 verifyNarrowIntegerArgs_Call(Outs, F: &MF.getFunction(), Callee);
2371
2372 // Analyze the operands of the call, assigning locations to each operand.
2373 SmallVector<CCValAssign, 16> ArgLocs;
2374 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2375 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2376
2377 // We don't support GuaranteedTailCallOpt, only automatically-detected
2378 // sibling calls.
2379 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2380 IsTailCall = false;
2381
2382 // Get a count of how many bytes are to be pushed on the stack.
2383 unsigned NumBytes = ArgCCInfo.getStackSize();
2384
2385 // Mark the start of the call.
2386 if (!IsTailCall)
2387 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2388
2389 // Copy argument values to their designated locations.
2390 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2391 SmallVector<SDValue, 8> MemOpChains;
2392 SDValue StackPtr;
2393 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2394 CCValAssign &VA = ArgLocs[I];
2395 SDValue ArgValue = OutVals[I];
2396
2397 if (VA.getLocInfo() == CCValAssign::Indirect) {
2398 // Store the argument in a stack slot and pass its address.
2399 EVT SlotVT;
2400 MVT PartVT;
2401 unsigned NumParts = 1;
2402 if (analyzeArgSplit(Args: Outs, ArgLocs, I, PartVT, NumParts))
2403 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * NumParts);
2404 else
2405 SlotVT = Outs[I].VT;
2406 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2407 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2408
2409 MachinePointerInfo StackPtrInfo =
2410 MachinePointerInfo::getFixedStack(MF, FI);
2411 MemOpChains.push_back(
2412 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot, PtrInfo: StackPtrInfo));
2413 // If the original argument was split (e.g. i128), we need
2414 // to store all parts of it here (and pass just one address).
2415 assert(Outs[I].PartOffset == 0);
2416 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2417 ++I;
2418 SDValue PartValue = OutVals[I];
2419 unsigned PartOffset = Outs[I].PartOffset;
2420 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2421 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2422 MemOpChains.push_back(
2423 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2424 PtrInfo: StackPtrInfo.getWithOffset(O: PartOffset)));
2425 assert(PartOffset && "Offset should be non-zero.");
2426 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2427 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2428 }
2429 ArgValue = SpillSlot;
2430 } else
2431 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2432
2433 if (VA.isRegLoc()) {
2434 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2435 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2436 // and low values.
2437 if (VA.getLocVT() == MVT::i128)
2438 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2439 // Queue up the argument copies and emit them at the end.
2440 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2441 } else {
2442 assert(VA.isMemLoc() && "Argument not register or memory");
2443
2444 // Work out the address of the stack slot. Unpromoted ints and
2445 // floats are passed as right-justified 8-byte values.
2446 if (!StackPtr.getNode())
2447 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2448 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2449 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2450 VA.getLocMemOffset();
2451 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2452 Offset += 4;
2453 else if (VA.getLocVT() == MVT::f16)
2454 Offset += 6;
2455 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2456 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2457
2458 // Emit the store.
2459 MemOpChains.push_back(
2460 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2461
2462 // Although long doubles or vectors are passed through the stack when
2463 // they are vararg (non-fixed arguments), if a long double or vector
2464 // occupies the third and fourth slot of the argument list GPR3 should
2465 // still shadow the third slot of the argument list.
2466 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2467 SDValue ShadowArgValue =
2468 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2469 N2: DAG.getIntPtrConstant(Val: 1, DL));
2470 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2471 }
2472 }
2473 }
2474
2475 // Join the stores, which are independent of one another.
2476 if (!MemOpChains.empty())
2477 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2478
2479 // Accept direct calls by converting symbolic call addresses to the
2480 // associated Target* opcodes. Force %r1 to be used for indirect
2481 // tail calls.
2482 SDValue Glue;
2483
2484 if (Subtarget.isTargetXPLINK64()) {
2485 SDValue ADA;
2486 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2487 if (!IsBRASL) {
2488 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2489 ->getAddressOfCalleeRegister();
2490 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2491 Glue = Chain.getValue(R: 1);
2492 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2493 }
2494 RegsToPass.push_back(Elt: std::make_pair(
2495 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2496 } else {
2497 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2498 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2499 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2500 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2501 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2502 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2503 } else if (IsTailCall) {
2504 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2505 Glue = Chain.getValue(R: 1);
2506 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2507 }
2508 }
2509
2510 // Build a sequence of copy-to-reg nodes, chained and glued together.
2511 for (const auto &[Reg, N] : RegsToPass) {
2512 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2513 Glue = Chain.getValue(R: 1);
2514 }
2515
2516 // The first call operand is the chain and the second is the target address.
2517 SmallVector<SDValue, 8> Ops;
2518 Ops.push_back(Elt: Chain);
2519 Ops.push_back(Elt: Callee);
2520
2521 // Add argument registers to the end of the list so that they are
2522 // known live into the call.
2523 for (const auto &[Reg, N] : RegsToPass)
2524 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2525
2526 // Add a register mask operand representing the call-preserved registers.
2527 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2528 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2529 assert(Mask && "Missing call preserved mask for calling convention");
2530 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2531
2532 // Glue the call to the argument copies, if any.
2533 if (Glue.getNode())
2534 Ops.push_back(Elt: Glue);
2535
2536 // Emit the call.
2537 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2538 if (IsTailCall) {
2539 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2540 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2541 return Ret;
2542 }
2543 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2544 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2545 Glue = Chain.getValue(R: 1);
2546
2547 // Mark the end of the call, which is glued to the call itself.
2548 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2549 Glue = Chain.getValue(R: 1);
2550
2551 // Assign locations to each value returned by this call.
2552 SmallVector<CCValAssign, 16> RetLocs;
2553 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2554 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2555
2556 // Copy all of the result registers out of their specified physreg.
2557 for (CCValAssign &VA : RetLocs) {
2558 // Copy the value out, gluing the copy to the end of the call sequence.
2559 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2560 VT: VA.getLocVT(), Glue);
2561 Chain = RetValue.getValue(R: 1);
2562 Glue = RetValue.getValue(R: 2);
2563
2564 // Convert the value of the return register into the value that's
2565 // being returned.
2566 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2567 }
2568
2569 return Chain;
2570}
2571
2572// Generate a call taking the given operands as arguments and returning a
2573// result of type RetVT.
2574std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2575 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2576 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2577 bool DoesNotReturn, bool IsReturnValueUsed) const {
2578 TargetLowering::ArgListTy Args;
2579 Args.reserve(n: Ops.size());
2580
2581 for (SDValue Op : Ops) {
2582 TargetLowering::ArgListEntry Entry(
2583 Op, Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
2584 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2585 Entry.IsZExt = !Entry.IsSExt;
2586 Args.push_back(x: Entry);
2587 }
2588
2589 SDValue Callee =
2590 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2591
2592 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2593 TargetLowering::CallLoweringInfo CLI(DAG);
2594 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2595 CLI.setDebugLoc(DL)
2596 .setChain(Chain)
2597 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2598 .setNoReturn(DoesNotReturn)
2599 .setDiscardResult(!IsReturnValueUsed)
2600 .setSExtResult(SignExtend)
2601 .setZExtResult(!SignExtend);
2602 return LowerCallTo(CLI);
2603}
2604
2605bool SystemZTargetLowering::CanLowerReturn(
2606 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2607 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2608 const Type *RetTy) const {
2609 // Special case that we cannot easily detect in RetCC_SystemZ since
2610 // i128 may not be a legal type.
2611 for (auto &Out : Outs)
2612 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2613 return false;
2614
2615 SmallVector<CCValAssign, 16> RetLocs;
2616 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2617 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2618}
2619
2620SDValue
2621SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2622 bool IsVarArg,
2623 const SmallVectorImpl<ISD::OutputArg> &Outs,
2624 const SmallVectorImpl<SDValue> &OutVals,
2625 const SDLoc &DL, SelectionDAG &DAG) const {
2626 MachineFunction &MF = DAG.getMachineFunction();
2627
2628 // Integer args <=32 bits should have an extension attribute.
2629 verifyNarrowIntegerArgs_Ret(Outs, F: &MF.getFunction());
2630
2631 // Assign locations to each returned value.
2632 SmallVector<CCValAssign, 16> RetLocs;
2633 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2634 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2635
2636 // Quick exit for void returns
2637 if (RetLocs.empty())
2638 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2639
2640 if (CallConv == CallingConv::GHC)
2641 report_fatal_error(reason: "GHC functions return void only");
2642
2643 // Copy the result values into the output registers.
2644 SDValue Glue;
2645 SmallVector<SDValue, 4> RetOps;
2646 RetOps.push_back(Elt: Chain);
2647 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2648 CCValAssign &VA = RetLocs[I];
2649 SDValue RetValue = OutVals[I];
2650
2651 // Make the return register live on exit.
2652 assert(VA.isRegLoc() && "Can only return in registers!");
2653
2654 // Promote the value as required.
2655 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2656
2657 // Chain and glue the copies together.
2658 Register Reg = VA.getLocReg();
2659 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2660 Glue = Chain.getValue(R: 1);
2661 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2662 }
2663
2664 // Update chain and glue.
2665 RetOps[0] = Chain;
2666 if (Glue.getNode())
2667 RetOps.push_back(Elt: Glue);
2668
2669 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2670}
2671
2672// Return true if Op is an intrinsic node with chain that returns the CC value
2673// as its only (other) argument. Provide the associated SystemZISD opcode and
2674// the mask of valid CC values if so.
2675static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2676 unsigned &CCValid) {
2677 unsigned Id = Op.getConstantOperandVal(i: 1);
2678 switch (Id) {
2679 case Intrinsic::s390_tbegin:
2680 Opcode = SystemZISD::TBEGIN;
2681 CCValid = SystemZ::CCMASK_TBEGIN;
2682 return true;
2683
2684 case Intrinsic::s390_tbegin_nofloat:
2685 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2686 CCValid = SystemZ::CCMASK_TBEGIN;
2687 return true;
2688
2689 case Intrinsic::s390_tend:
2690 Opcode = SystemZISD::TEND;
2691 CCValid = SystemZ::CCMASK_TEND;
2692 return true;
2693
2694 default:
2695 return false;
2696 }
2697}
2698
2699// Return true if Op is an intrinsic node without chain that returns the
2700// CC value as its final argument. Provide the associated SystemZISD
2701// opcode and the mask of valid CC values if so.
2702static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2703 unsigned Id = Op.getConstantOperandVal(i: 0);
2704 switch (Id) {
2705 case Intrinsic::s390_vpkshs:
2706 case Intrinsic::s390_vpksfs:
2707 case Intrinsic::s390_vpksgs:
2708 Opcode = SystemZISD::PACKS_CC;
2709 CCValid = SystemZ::CCMASK_VCMP;
2710 return true;
2711
2712 case Intrinsic::s390_vpklshs:
2713 case Intrinsic::s390_vpklsfs:
2714 case Intrinsic::s390_vpklsgs:
2715 Opcode = SystemZISD::PACKLS_CC;
2716 CCValid = SystemZ::CCMASK_VCMP;
2717 return true;
2718
2719 case Intrinsic::s390_vceqbs:
2720 case Intrinsic::s390_vceqhs:
2721 case Intrinsic::s390_vceqfs:
2722 case Intrinsic::s390_vceqgs:
2723 case Intrinsic::s390_vceqqs:
2724 Opcode = SystemZISD::VICMPES;
2725 CCValid = SystemZ::CCMASK_VCMP;
2726 return true;
2727
2728 case Intrinsic::s390_vchbs:
2729 case Intrinsic::s390_vchhs:
2730 case Intrinsic::s390_vchfs:
2731 case Intrinsic::s390_vchgs:
2732 case Intrinsic::s390_vchqs:
2733 Opcode = SystemZISD::VICMPHS;
2734 CCValid = SystemZ::CCMASK_VCMP;
2735 return true;
2736
2737 case Intrinsic::s390_vchlbs:
2738 case Intrinsic::s390_vchlhs:
2739 case Intrinsic::s390_vchlfs:
2740 case Intrinsic::s390_vchlgs:
2741 case Intrinsic::s390_vchlqs:
2742 Opcode = SystemZISD::VICMPHLS;
2743 CCValid = SystemZ::CCMASK_VCMP;
2744 return true;
2745
2746 case Intrinsic::s390_vtm:
2747 Opcode = SystemZISD::VTM;
2748 CCValid = SystemZ::CCMASK_VCMP;
2749 return true;
2750
2751 case Intrinsic::s390_vfaebs:
2752 case Intrinsic::s390_vfaehs:
2753 case Intrinsic::s390_vfaefs:
2754 Opcode = SystemZISD::VFAE_CC;
2755 CCValid = SystemZ::CCMASK_ANY;
2756 return true;
2757
2758 case Intrinsic::s390_vfaezbs:
2759 case Intrinsic::s390_vfaezhs:
2760 case Intrinsic::s390_vfaezfs:
2761 Opcode = SystemZISD::VFAEZ_CC;
2762 CCValid = SystemZ::CCMASK_ANY;
2763 return true;
2764
2765 case Intrinsic::s390_vfeebs:
2766 case Intrinsic::s390_vfeehs:
2767 case Intrinsic::s390_vfeefs:
2768 Opcode = SystemZISD::VFEE_CC;
2769 CCValid = SystemZ::CCMASK_ANY;
2770 return true;
2771
2772 case Intrinsic::s390_vfeezbs:
2773 case Intrinsic::s390_vfeezhs:
2774 case Intrinsic::s390_vfeezfs:
2775 Opcode = SystemZISD::VFEEZ_CC;
2776 CCValid = SystemZ::CCMASK_ANY;
2777 return true;
2778
2779 case Intrinsic::s390_vfenebs:
2780 case Intrinsic::s390_vfenehs:
2781 case Intrinsic::s390_vfenefs:
2782 Opcode = SystemZISD::VFENE_CC;
2783 CCValid = SystemZ::CCMASK_ANY;
2784 return true;
2785
2786 case Intrinsic::s390_vfenezbs:
2787 case Intrinsic::s390_vfenezhs:
2788 case Intrinsic::s390_vfenezfs:
2789 Opcode = SystemZISD::VFENEZ_CC;
2790 CCValid = SystemZ::CCMASK_ANY;
2791 return true;
2792
2793 case Intrinsic::s390_vistrbs:
2794 case Intrinsic::s390_vistrhs:
2795 case Intrinsic::s390_vistrfs:
2796 Opcode = SystemZISD::VISTR_CC;
2797 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2798 return true;
2799
2800 case Intrinsic::s390_vstrcbs:
2801 case Intrinsic::s390_vstrchs:
2802 case Intrinsic::s390_vstrcfs:
2803 Opcode = SystemZISD::VSTRC_CC;
2804 CCValid = SystemZ::CCMASK_ANY;
2805 return true;
2806
2807 case Intrinsic::s390_vstrczbs:
2808 case Intrinsic::s390_vstrczhs:
2809 case Intrinsic::s390_vstrczfs:
2810 Opcode = SystemZISD::VSTRCZ_CC;
2811 CCValid = SystemZ::CCMASK_ANY;
2812 return true;
2813
2814 case Intrinsic::s390_vstrsb:
2815 case Intrinsic::s390_vstrsh:
2816 case Intrinsic::s390_vstrsf:
2817 Opcode = SystemZISD::VSTRS_CC;
2818 CCValid = SystemZ::CCMASK_ANY;
2819 return true;
2820
2821 case Intrinsic::s390_vstrszb:
2822 case Intrinsic::s390_vstrszh:
2823 case Intrinsic::s390_vstrszf:
2824 Opcode = SystemZISD::VSTRSZ_CC;
2825 CCValid = SystemZ::CCMASK_ANY;
2826 return true;
2827
2828 case Intrinsic::s390_vfcedbs:
2829 case Intrinsic::s390_vfcesbs:
2830 Opcode = SystemZISD::VFCMPES;
2831 CCValid = SystemZ::CCMASK_VCMP;
2832 return true;
2833
2834 case Intrinsic::s390_vfchdbs:
2835 case Intrinsic::s390_vfchsbs:
2836 Opcode = SystemZISD::VFCMPHS;
2837 CCValid = SystemZ::CCMASK_VCMP;
2838 return true;
2839
2840 case Intrinsic::s390_vfchedbs:
2841 case Intrinsic::s390_vfchesbs:
2842 Opcode = SystemZISD::VFCMPHES;
2843 CCValid = SystemZ::CCMASK_VCMP;
2844 return true;
2845
2846 case Intrinsic::s390_vftcidb:
2847 case Intrinsic::s390_vftcisb:
2848 Opcode = SystemZISD::VFTCI;
2849 CCValid = SystemZ::CCMASK_VCMP;
2850 return true;
2851
2852 case Intrinsic::s390_tdc:
2853 Opcode = SystemZISD::TDC;
2854 CCValid = SystemZ::CCMASK_TDC;
2855 return true;
2856
2857 default:
2858 return false;
2859 }
2860}
2861
2862// Emit an intrinsic with chain and an explicit CC register result.
2863static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2864 unsigned Opcode) {
2865 // Copy all operands except the intrinsic ID.
2866 unsigned NumOps = Op.getNumOperands();
2867 SmallVector<SDValue, 6> Ops;
2868 Ops.reserve(N: NumOps - 1);
2869 Ops.push_back(Elt: Op.getOperand(i: 0));
2870 for (unsigned I = 2; I < NumOps; ++I)
2871 Ops.push_back(Elt: Op.getOperand(i: I));
2872
2873 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2874 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2875 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2876 SDValue OldChain = SDValue(Op.getNode(), 1);
2877 SDValue NewChain = SDValue(Intr.getNode(), 1);
2878 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2879 return Intr.getNode();
2880}
2881
2882// Emit an intrinsic with an explicit CC register result.
2883static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2884 unsigned Opcode) {
2885 // Copy all operands except the intrinsic ID.
2886 SDLoc DL(Op);
2887 unsigned NumOps = Op.getNumOperands();
2888 SmallVector<SDValue, 6> Ops;
2889 Ops.reserve(N: NumOps - 1);
2890 for (unsigned I = 1; I < NumOps; ++I) {
2891 SDValue CurrOper = Op.getOperand(i: I);
2892 if (CurrOper.getValueType() == MVT::f16) {
2893 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2894 "Unhandled intrinsic with f16 operand.");
2895 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2896 }
2897 Ops.push_back(Elt: CurrOper);
2898 }
2899
2900 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2901 return Intr.getNode();
2902}
2903
2904// CC is a comparison that will be implemented using an integer or
2905// floating-point comparison. Return the condition code mask for
2906// a branch on true. In the integer case, CCMASK_CMP_UO is set for
2907// unsigned comparisons and clear for signed ones. In the floating-point
2908// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
2909static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2910#define CONV(X) \
2911 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2912 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2913 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2914
2915 switch (CC) {
2916 default:
2917 llvm_unreachable("Invalid integer condition!");
2918
2919 CONV(EQ);
2920 CONV(NE);
2921 CONV(GT);
2922 CONV(GE);
2923 CONV(LT);
2924 CONV(LE);
2925
2926 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2927 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2928 }
2929#undef CONV
2930}
2931
2932// If C can be converted to a comparison against zero, adjust the operands
2933// as necessary.
2934static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2935 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2936 return;
2937
2938 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
2939 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2940 return;
2941
2942 int64_t Value = ConstOp1->getSExtValue();
2943 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2944 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2945 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2946 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2947 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2948 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
2949 }
2950}
2951
2952// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2953// adjust the operands as necessary.
2954static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2955 Comparison &C) {
2956 // For us to make any changes, it must a comparison between a single-use
2957 // load and a constant.
2958 if (!C.Op0.hasOneUse() ||
2959 C.Op0.getOpcode() != ISD::LOAD ||
2960 C.Op1.getOpcode() != ISD::Constant)
2961 return;
2962
2963 // We must have an 8- or 16-bit load.
2964 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
2965 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2966 if ((NumBits != 8 && NumBits != 16) ||
2967 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2968 return;
2969
2970 // The load must be an extending one and the constant must be within the
2971 // range of the unextended value.
2972 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
2973 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2974 return;
2975 uint64_t Value = ConstOp1->getZExtValue();
2976 uint64_t Mask = (1 << NumBits) - 1;
2977 if (Load->getExtensionType() == ISD::SEXTLOAD) {
2978 // Make sure that ConstOp1 is in range of C.Op0.
2979 int64_t SignedValue = ConstOp1->getSExtValue();
2980 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
2981 return;
2982 if (C.ICmpType != SystemZICMP::SignedOnly) {
2983 // Unsigned comparison between two sign-extended values is equivalent
2984 // to unsigned comparison between two zero-extended values.
2985 Value &= Mask;
2986 } else if (NumBits == 8) {
2987 // Try to treat the comparison as unsigned, so that we can use CLI.
2988 // Adjust CCMask and Value as necessary.
2989 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
2990 // Test whether the high bit of the byte is set.
2991 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
2992 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
2993 // Test whether the high bit of the byte is clear.
2994 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
2995 else
2996 // No instruction exists for this combination.
2997 return;
2998 C.ICmpType = SystemZICMP::UnsignedOnly;
2999 }
3000 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
3001 if (Value > Mask)
3002 return;
3003 // If the constant is in range, we can use any comparison.
3004 C.ICmpType = SystemZICMP::Any;
3005 } else
3006 return;
3007
3008 // Make sure that the first operand is an i32 of the right extension type.
3009 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
3010 ISD::SEXTLOAD :
3011 ISD::ZEXTLOAD);
3012 if (C.Op0.getValueType() != MVT::i32 ||
3013 Load->getExtensionType() != ExtType) {
3014 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
3015 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
3016 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
3017 MMOFlags: Load->getMemOperand()->getFlags());
3018 // Update the chain uses.
3019 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
3020 }
3021
3022 // Make sure that the second operand is an i32 with the right value.
3023 if (C.Op1.getValueType() != MVT::i32 ||
3024 Value != ConstOp1->getZExtValue())
3025 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
3026}
3027
3028// Return true if Op is either an unextended load, or a load suitable
3029// for integer register-memory comparisons of type ICmpType.
3030static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3031 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
3032 if (Load) {
3033 // There are no instructions to compare a register with a memory byte.
3034 if (Load->getMemoryVT() == MVT::i8)
3035 return false;
3036 // Otherwise decide on extension type.
3037 switch (Load->getExtensionType()) {
3038 case ISD::NON_EXTLOAD:
3039 return true;
3040 case ISD::SEXTLOAD:
3041 return ICmpType != SystemZICMP::UnsignedOnly;
3042 case ISD::ZEXTLOAD:
3043 return ICmpType != SystemZICMP::SignedOnly;
3044 default:
3045 break;
3046 }
3047 }
3048 return false;
3049}
3050
3051// Return true if it is better to swap the operands of C.
3052static bool shouldSwapCmpOperands(const Comparison &C) {
3053 // If one side of the compare is a load of the stackguard reference value,
3054 // then that load should be Op1.
3055 if (C.Op0.isMachineOpcode() &&
3056 (C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3057 return true;
3058
3059 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3060 if (C.Op0.getValueType() == MVT::i128)
3061 return false;
3062 if (C.Op0.getValueType() == MVT::f128)
3063 return false;
3064
3065 // Always keep a floating-point constant second, since comparisons with
3066 // zero can use LOAD TEST and comparisons with other constants make a
3067 // natural memory operand.
3068 if (isa<ConstantFPSDNode>(Val: C.Op1))
3069 return false;
3070
3071 // Never swap comparisons with zero since there are many ways to optimize
3072 // those later.
3073 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
3074 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3075 return false;
3076
3077 // Also keep natural memory operands second if the loaded value is
3078 // only used here. Several comparisons have memory forms.
3079 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
3080 return false;
3081
3082 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3083 // In that case we generally prefer the memory to be second.
3084 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
3085 // The only exceptions are when the second operand is a constant and
3086 // we can use things like CHHSI.
3087 if (!ConstOp1)
3088 return true;
3089 // The unsigned memory-immediate instructions can handle 16-bit
3090 // unsigned integers.
3091 if (C.ICmpType != SystemZICMP::SignedOnly &&
3092 isUInt<16>(x: ConstOp1->getZExtValue()))
3093 return false;
3094 // The signed memory-immediate instructions can handle 16-bit
3095 // signed integers.
3096 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3097 isInt<16>(x: ConstOp1->getSExtValue()))
3098 return false;
3099 return true;
3100 }
3101
3102 // Try to promote the use of CGFR and CLGFR.
3103 unsigned Opcode0 = C.Op0.getOpcode();
3104 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3105 return true;
3106 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3107 return true;
3108 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3109 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
3110 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
3111 return true;
3112
3113 return false;
3114}
3115
3116// Check whether C tests for equality between X and Y and whether X - Y
3117// or Y - X is also computed. In that case it's better to compare the
3118// result of the subtraction against zero.
3119static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
3120 Comparison &C) {
3121 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3122 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3123 for (SDNode *N : C.Op0->users()) {
3124 if (N->getOpcode() == ISD::SUB &&
3125 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3126 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3127 // Disable the nsw and nuw flags: the backend needs to handle
3128 // overflow as well during comparison elimination.
3129 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3130 C.Op0 = SDValue(N, 0);
3131 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3132 return;
3133 }
3134 }
3135 }
3136}
3137
3138// Check whether C compares a floating-point value with zero and if that
3139// floating-point value is also negated. In this case we can use the
3140// negation to set CC, so avoiding separate LOAD AND TEST and
3141// LOAD (NEGATIVE/COMPLEMENT) instructions.
3142static void adjustForFNeg(Comparison &C) {
3143 // This optimization is invalid for strict comparisons, since FNEG
3144 // does not raise any exceptions.
3145 if (C.Chain)
3146 return;
3147 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3148 if (C1 && C1->isZero()) {
3149 for (SDNode *N : C.Op0->users()) {
3150 if (N->getOpcode() == ISD::FNEG) {
3151 C.Op0 = SDValue(N, 0);
3152 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3153 return;
3154 }
3155 }
3156 }
3157}
3158
3159// Check whether C compares (shl X, 32) with 0 and whether X is
3160// also sign-extended. In that case it is better to test the result
3161// of the sign extension using LTGFR.
3162//
3163// This case is important because InstCombine transforms a comparison
3164// with (sext (trunc X)) into a comparison with (shl X, 32).
3165static void adjustForLTGFR(Comparison &C) {
3166 // Check for a comparison between (shl X, 32) and 0.
3167 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3168 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3169 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3170 if (C1 && C1->getZExtValue() == 32) {
3171 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3172 // See whether X has any SIGN_EXTEND_INREG uses.
3173 for (SDNode *N : ShlOp0->users()) {
3174 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3175 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3176 C.Op0 = SDValue(N, 0);
3177 return;
3178 }
3179 }
3180 }
3181 }
3182}
3183
3184// If C compares the truncation of an extending load, try to compare
3185// the untruncated value instead. This exposes more opportunities to
3186// reuse CC.
3187static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3188 Comparison &C) {
3189 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3190 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3191 C.Op1.getOpcode() == ISD::Constant &&
3192 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3193 C.Op1->getAsZExtVal() == 0) {
3194 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3195 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3196 C.Op0.getValueSizeInBits().getFixedValue()) {
3197 unsigned Type = L->getExtensionType();
3198 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3199 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3200 C.Op0 = C.Op0.getOperand(i: 0);
3201 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3202 }
3203 }
3204 }
3205}
3206
3207// Adjust if a given Compare is a check of the stack guard against a stack
3208// guard instance on the stack. Specifically, this checks if:
3209// - The operands are a load of the stack guard, and a load from a stack slot
3210// - The original opcode is ICMP
3211// - ICMPType is compatible with unsigned comparison.
3212static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL,
3213 Comparison &C) {
3214
3215 // Opcode must be ICMP.
3216 if (C.Opcode != SystemZISD::ICMP)
3217 return;
3218 // ICmpType must be Unsigned or Any.
3219 if (C.ICmpType == SystemZICMP::SignedOnly)
3220 return;
3221 // Op0 must be FrameIndex Load.
3222 if (!(ISD::isNormalLoad(N: C.Op0.getNode()) &&
3223 dyn_cast<FrameIndexSDNode>(Val: C.Op0.getOperand(i: 1))))
3224 return;
3225 // Op1 must be LOAD_STACK_GUARD.
3226 if (!C.Op1.isMachineOpcode() ||
3227 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3228 return;
3229
3230 // At this point we are sure that this is a proper CMP_STACKGUARD
3231 // case, update the opcode to reflect this.
3232 C.Opcode = SystemZISD::CMP_STACKGUARD;
3233 C.Op1 = SDValue();
3234}
3235
3236// Return true if shift operation N has an in-range constant shift value.
3237// Store it in ShiftVal if so.
3238static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3239 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3240 if (!Shift)
3241 return false;
3242
3243 uint64_t Amount = Shift->getZExtValue();
3244 if (Amount >= N.getValueSizeInBits())
3245 return false;
3246
3247 ShiftVal = Amount;
3248 return true;
3249}
3250
3251// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3252// instruction and whether the CC value is descriptive enough to handle
3253// a comparison of type Opcode between the AND result and CmpVal.
3254// CCMask says which comparison result is being tested and BitSize is
3255// the number of bits in the operands. If TEST UNDER MASK can be used,
3256// return the corresponding CC mask, otherwise return 0.
3257static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3258 uint64_t Mask, uint64_t CmpVal,
3259 unsigned ICmpType) {
3260 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3261
3262 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3263 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3264 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3265 return 0;
3266
3267 // Work out the masks for the lowest and highest bits.
3268 uint64_t High = llvm::bit_floor(Value: Mask);
3269 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3270
3271 // Signed ordered comparisons are effectively unsigned if the sign
3272 // bit is dropped.
3273 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3274
3275 // Check for equality comparisons with 0, or the equivalent.
3276 if (CmpVal == 0) {
3277 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3278 return SystemZ::CCMASK_TM_ALL_0;
3279 if (CCMask == SystemZ::CCMASK_CMP_NE)
3280 return SystemZ::CCMASK_TM_SOME_1;
3281 }
3282 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3283 if (CCMask == SystemZ::CCMASK_CMP_LT)
3284 return SystemZ::CCMASK_TM_ALL_0;
3285 if (CCMask == SystemZ::CCMASK_CMP_GE)
3286 return SystemZ::CCMASK_TM_SOME_1;
3287 }
3288 if (EffectivelyUnsigned && CmpVal < Low) {
3289 if (CCMask == SystemZ::CCMASK_CMP_LE)
3290 return SystemZ::CCMASK_TM_ALL_0;
3291 if (CCMask == SystemZ::CCMASK_CMP_GT)
3292 return SystemZ::CCMASK_TM_SOME_1;
3293 }
3294
3295 // Check for equality comparisons with the mask, or the equivalent.
3296 if (CmpVal == Mask) {
3297 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3298 return SystemZ::CCMASK_TM_ALL_1;
3299 if (CCMask == SystemZ::CCMASK_CMP_NE)
3300 return SystemZ::CCMASK_TM_SOME_0;
3301 }
3302 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3303 if (CCMask == SystemZ::CCMASK_CMP_GT)
3304 return SystemZ::CCMASK_TM_ALL_1;
3305 if (CCMask == SystemZ::CCMASK_CMP_LE)
3306 return SystemZ::CCMASK_TM_SOME_0;
3307 }
3308 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3309 if (CCMask == SystemZ::CCMASK_CMP_GE)
3310 return SystemZ::CCMASK_TM_ALL_1;
3311 if (CCMask == SystemZ::CCMASK_CMP_LT)
3312 return SystemZ::CCMASK_TM_SOME_0;
3313 }
3314
3315 // Check for ordered comparisons with the top bit.
3316 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3317 if (CCMask == SystemZ::CCMASK_CMP_LE)
3318 return SystemZ::CCMASK_TM_MSB_0;
3319 if (CCMask == SystemZ::CCMASK_CMP_GT)
3320 return SystemZ::CCMASK_TM_MSB_1;
3321 }
3322 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3323 if (CCMask == SystemZ::CCMASK_CMP_LT)
3324 return SystemZ::CCMASK_TM_MSB_0;
3325 if (CCMask == SystemZ::CCMASK_CMP_GE)
3326 return SystemZ::CCMASK_TM_MSB_1;
3327 }
3328
3329 // If there are just two bits, we can do equality checks for Low and High
3330 // as well.
3331 if (Mask == Low + High) {
3332 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3333 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3334 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3335 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3336 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3337 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3338 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3339 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3340 }
3341
3342 // Looks like we've exhausted our options.
3343 return 0;
3344}
3345
3346// See whether C can be implemented as a TEST UNDER MASK instruction.
3347// Update the arguments with the TM version if so.
3348static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3349 Comparison &C) {
3350 // Use VECTOR TEST UNDER MASK for i128 operations.
3351 if (C.Op0.getValueType() == MVT::i128) {
3352 // We can use VTM for EQ/NE comparisons of x & y against 0.
3353 if (C.Op0.getOpcode() == ISD::AND &&
3354 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3355 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3356 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3357 if (Mask && Mask->getAPIntValue() == 0) {
3358 C.Opcode = SystemZISD::VTM;
3359 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3360 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3361 C.CCValid = SystemZ::CCMASK_VCMP;
3362 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3363 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3364 else
3365 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3366 }
3367 }
3368 return;
3369 }
3370
3371 // Check that we have a comparison with a constant.
3372 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3373 if (!ConstOp1)
3374 return;
3375 uint64_t CmpVal = ConstOp1->getZExtValue();
3376
3377 // Check whether the nonconstant input is an AND with a constant mask.
3378 Comparison NewC(C);
3379 uint64_t MaskVal;
3380 ConstantSDNode *Mask = nullptr;
3381 if (C.Op0.getOpcode() == ISD::AND) {
3382 NewC.Op0 = C.Op0.getOperand(i: 0);
3383 NewC.Op1 = C.Op0.getOperand(i: 1);
3384 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3385 if (!Mask)
3386 return;
3387 MaskVal = Mask->getZExtValue();
3388 } else {
3389 // There is no instruction to compare with a 64-bit immediate
3390 // so use TMHH instead if possible. We need an unsigned ordered
3391 // comparison with an i64 immediate.
3392 if (NewC.Op0.getValueType() != MVT::i64 ||
3393 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3394 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3395 NewC.ICmpType == SystemZICMP::SignedOnly)
3396 return;
3397 // Convert LE and GT comparisons into LT and GE.
3398 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3399 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3400 if (CmpVal == uint64_t(-1))
3401 return;
3402 CmpVal += 1;
3403 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3404 }
3405 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3406 // be masked off without changing the result.
3407 MaskVal = -(CmpVal & -CmpVal);
3408 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3409 }
3410 if (!MaskVal)
3411 return;
3412
3413 // Check whether the combination of mask, comparison value and comparison
3414 // type are suitable.
3415 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3416 unsigned NewCCMask, ShiftVal;
3417 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3418 NewC.Op0.getOpcode() == ISD::SHL &&
3419 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3420 (MaskVal >> ShiftVal != 0) &&
3421 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3422 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3423 Mask: MaskVal >> ShiftVal,
3424 CmpVal: CmpVal >> ShiftVal,
3425 ICmpType: SystemZICMP::Any))) {
3426 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3427 MaskVal >>= ShiftVal;
3428 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3429 NewC.Op0.getOpcode() == ISD::SRL &&
3430 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3431 (MaskVal << ShiftVal != 0) &&
3432 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3433 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3434 Mask: MaskVal << ShiftVal,
3435 CmpVal: CmpVal << ShiftVal,
3436 ICmpType: SystemZICMP::UnsignedOnly))) {
3437 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3438 MaskVal <<= ShiftVal;
3439 } else {
3440 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3441 ICmpType: NewC.ICmpType);
3442 if (!NewCCMask)
3443 return;
3444 }
3445
3446 // Go ahead and make the change.
3447 C.Opcode = SystemZISD::TM;
3448 C.Op0 = NewC.Op0;
3449 if (Mask && Mask->getZExtValue() == MaskVal)
3450 C.Op1 = SDValue(Mask, 0);
3451 else
3452 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3453 C.CCValid = SystemZ::CCMASK_TM;
3454 C.CCMask = NewCCMask;
3455}
3456
3457// Implement i128 comparison in vector registers.
3458static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3459 Comparison &C) {
3460 if (C.Opcode != SystemZISD::ICMP)
3461 return;
3462 if (C.Op0.getValueType() != MVT::i128)
3463 return;
3464
3465 // Recognize vector comparison reductions.
3466 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3467 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3468 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3469 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3470 bool CmpNull = isNullConstant(V: C.Op1);
3471 SDValue Src = peekThroughBitcasts(V: C.Op0);
3472 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3473 Src = Src.getOperand(i: 0);
3474 CmpNull = !CmpNull;
3475 }
3476 unsigned Opcode = 0;
3477 if (Src.hasOneUse()) {
3478 switch (Src.getOpcode()) {
3479 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3480 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3481 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3482 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3483 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3484 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3485 default: break;
3486 }
3487 }
3488 if (Opcode) {
3489 C.Opcode = Opcode;
3490 C.Op0 = Src->getOperand(Num: 0);
3491 C.Op1 = Src->getOperand(Num: 1);
3492 C.CCValid = SystemZ::CCMASK_VCMP;
3493 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3494 if (!CmpEq)
3495 C.CCMask ^= C.CCValid;
3496 return;
3497 }
3498 }
3499
3500 // Everything below here is not useful if we have native i128 compares.
3501 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3502 return;
3503
3504 // (In-)Equality comparisons can be implemented via VCEQGS.
3505 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3506 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3507 C.Opcode = SystemZISD::VICMPES;
3508 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3509 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3510 C.CCValid = SystemZ::CCMASK_VCMP;
3511 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3512 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3513 else
3514 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3515 return;
3516 }
3517
3518 // Normalize other comparisons to GT.
3519 bool Swap = false, Invert = false;
3520 switch (C.CCMask) {
3521 case SystemZ::CCMASK_CMP_GT: break;
3522 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3523 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3524 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3525 default: llvm_unreachable("Invalid integer condition!");
3526 }
3527 if (Swap)
3528 std::swap(a&: C.Op0, b&: C.Op1);
3529
3530 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3531 C.Opcode = SystemZISD::UCMP128HI;
3532 else
3533 C.Opcode = SystemZISD::SCMP128HI;
3534 C.CCValid = SystemZ::CCMASK_ANY;
3535 C.CCMask = SystemZ::CCMASK_1;
3536
3537 if (Invert)
3538 C.CCMask ^= C.CCValid;
3539}
3540
3541// See whether the comparison argument contains a redundant AND
3542// and remove it if so. This sometimes happens due to the generic
3543// BRCOND expansion.
3544static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3545 Comparison &C) {
3546 if (C.Op0.getOpcode() != ISD::AND)
3547 return;
3548 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3549 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3550 return;
3551 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3552 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3553 return;
3554
3555 C.Op0 = C.Op0.getOperand(i: 0);
3556}
3557
3558// Return a Comparison that tests the condition-code result of intrinsic
3559// node Call against constant integer CC using comparison code Cond.
3560// Opcode is the opcode of the SystemZISD operation for the intrinsic
3561// and CCValid is the set of possible condition-code results.
3562static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3563 SDValue Call, unsigned CCValid, uint64_t CC,
3564 ISD::CondCode Cond) {
3565 Comparison C(Call, SDValue(), SDValue());
3566 C.Opcode = Opcode;
3567 C.CCValid = CCValid;
3568 if (Cond == ISD::SETEQ)
3569 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3570 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3571 else if (Cond == ISD::SETNE)
3572 // ...and the inverse of that.
3573 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3574 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3575 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3576 // always true for CC>3.
3577 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3578 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3579 // ...and the inverse of that.
3580 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3581 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3582 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3583 // always true for CC>3.
3584 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3585 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3586 // ...and the inverse of that.
3587 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3588 else
3589 llvm_unreachable("Unexpected integer comparison type");
3590 C.CCMask &= CCValid;
3591 return C;
3592}
3593
3594// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3595static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3596 ISD::CondCode Cond, const SDLoc &DL,
3597 SDValue Chain = SDValue(),
3598 bool IsSignaling = false) {
3599 if (CmpOp1.getOpcode() == ISD::Constant) {
3600 assert(!Chain);
3601 unsigned Opcode, CCValid;
3602 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3603 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3604 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3605 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3606 CC: CmpOp1->getAsZExtVal(), Cond);
3607 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3608 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3609 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3610 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3611 CC: CmpOp1->getAsZExtVal(), Cond);
3612 }
3613 Comparison C(CmpOp0, CmpOp1, Chain);
3614 C.CCMask = CCMaskForCondCode(CC: Cond);
3615 if (C.Op0.getValueType().isFloatingPoint()) {
3616 C.CCValid = SystemZ::CCMASK_FCMP;
3617 if (!C.Chain)
3618 C.Opcode = SystemZISD::FCMP;
3619 else if (!IsSignaling)
3620 C.Opcode = SystemZISD::STRICT_FCMP;
3621 else
3622 C.Opcode = SystemZISD::STRICT_FCMPS;
3623 adjustForFNeg(C);
3624 } else {
3625 assert(!C.Chain);
3626 C.CCValid = SystemZ::CCMASK_ICMP;
3627 C.Opcode = SystemZISD::ICMP;
3628 // Choose the type of comparison. Equality and inequality tests can
3629 // use either signed or unsigned comparisons. The choice also doesn't
3630 // matter if both sign bits are known to be clear. In those cases we
3631 // want to give the main isel code the freedom to choose whichever
3632 // form fits best.
3633 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3634 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3635 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3636 C.ICmpType = SystemZICMP::Any;
3637 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3638 C.ICmpType = SystemZICMP::UnsignedOnly;
3639 else
3640 C.ICmpType = SystemZICMP::SignedOnly;
3641 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3642 adjustForRedundantAnd(DAG, DL, C);
3643 adjustZeroCmp(DAG, DL, C);
3644 adjustSubwordCmp(DAG, DL, C);
3645 adjustForSubtraction(DAG, DL, C);
3646 adjustForLTGFR(C);
3647 adjustICmpTruncate(DAG, DL, C);
3648 }
3649
3650 if (shouldSwapCmpOperands(C)) {
3651 std::swap(a&: C.Op0, b&: C.Op1);
3652 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3653 }
3654
3655 adjustForTestUnderMask(DAG, DL, C);
3656 adjustICmp128(DAG, DL, C);
3657 adjustForStackGuardCompare(DAG, DL, C);
3658 return C;
3659}
3660
3661// Emit the comparison instruction described by C.
3662static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3663 if (!C.Op1.getNode()) {
3664 if (C.Opcode == SystemZISD::CMP_STACKGUARD)
3665 return DAG.getNode(Opcode: SystemZISD::CMP_STACKGUARD, DL, VT: MVT::i32, Operand: C.Op0);
3666 SDNode *Node;
3667 switch (C.Op0.getOpcode()) {
3668 case ISD::INTRINSIC_W_CHAIN:
3669 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3670 return SDValue(Node, 0);
3671 case ISD::INTRINSIC_WO_CHAIN:
3672 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3673 return SDValue(Node, Node->getNumValues() - 1);
3674 default:
3675 llvm_unreachable("Invalid comparison operands");
3676 }
3677 }
3678 if (C.Opcode == SystemZISD::ICMP)
3679 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3680 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3681 if (C.Opcode == SystemZISD::TM) {
3682 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3683 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3684 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3685 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3686 }
3687 if (C.Opcode == SystemZISD::VICMPES ||
3688 C.Opcode == SystemZISD::VICMPHS ||
3689 C.Opcode == SystemZISD::VICMPHLS ||
3690 C.Opcode == SystemZISD::VFCMPES ||
3691 C.Opcode == SystemZISD::VFCMPHS ||
3692 C.Opcode == SystemZISD::VFCMPHES) {
3693 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3694 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3695 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3696 return SDValue(Val.getNode(), 1);
3697 }
3698 if (C.Chain) {
3699 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3700 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3701 }
3702 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3703}
3704
3705// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3706// 64 bits. Extend is the extension type to use. Store the high part
3707// in Hi and the low part in Lo.
3708static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3709 SDValue Op0, SDValue Op1, SDValue &Hi,
3710 SDValue &Lo) {
3711 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3712 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3713 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3714 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3715 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3716 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3717 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3718}
3719
3720// Lower a binary operation that produces two VT results, one in each
3721// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3722// and Opcode performs the GR128 operation. Store the even register result
3723// in Even and the odd register result in Odd.
3724static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3725 unsigned Opcode, SDValue Op0, SDValue Op1,
3726 SDValue &Even, SDValue &Odd) {
3727 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::Untyped, N1: Op0, N2: Op1);
3728 bool Is32Bit = is32Bit(VT);
3729 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3730 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3731}
3732
3733// Return an i32 value that is 1 if the CC value produced by CCReg is
3734// in the mask CCMask and 0 otherwise. CC is known to have a value
3735// in CCValid, so other values can be ignored.
3736static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3737 unsigned CCValid, unsigned CCMask) {
3738 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3739 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3740 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3741 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3742 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3743}
3744
3745// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3746// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3747// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3748// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3749// floating-point comparisons.
3750enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3751static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3752 switch (CC) {
3753 case ISD::SETOEQ:
3754 case ISD::SETEQ:
3755 switch (Mode) {
3756 case CmpMode::Int: return SystemZISD::VICMPE;
3757 case CmpMode::FP: return SystemZISD::VFCMPE;
3758 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3759 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3760 }
3761 llvm_unreachable("Bad mode");
3762
3763 case ISD::SETOGE:
3764 case ISD::SETGE:
3765 switch (Mode) {
3766 case CmpMode::Int: return 0;
3767 case CmpMode::FP: return SystemZISD::VFCMPHE;
3768 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3769 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3770 }
3771 llvm_unreachable("Bad mode");
3772
3773 case ISD::SETOGT:
3774 case ISD::SETGT:
3775 switch (Mode) {
3776 case CmpMode::Int: return SystemZISD::VICMPH;
3777 case CmpMode::FP: return SystemZISD::VFCMPH;
3778 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3779 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3780 }
3781 llvm_unreachable("Bad mode");
3782
3783 case ISD::SETUGT:
3784 switch (Mode) {
3785 case CmpMode::Int: return SystemZISD::VICMPHL;
3786 case CmpMode::FP: return 0;
3787 case CmpMode::StrictFP: return 0;
3788 case CmpMode::SignalingFP: return 0;
3789 }
3790 llvm_unreachable("Bad mode");
3791
3792 default:
3793 return 0;
3794 }
3795}
3796
3797// Return the SystemZISD vector comparison operation for CC or its inverse,
3798// or 0 if neither can be done directly. Indicate in Invert whether the
3799// result is for the inverse of CC. Mode is as above.
3800static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3801 bool &Invert) {
3802 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3803 Invert = false;
3804 return Opcode;
3805 }
3806
3807 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3808 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3809 Invert = true;
3810 return Opcode;
3811 }
3812
3813 return 0;
3814}
3815
3816// Return a v2f64 that contains the extended form of elements Start and Start+1
3817// of v4f32 value Op. If Chain is nonnull, return the strict form.
3818static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3819 SDValue Op, SDValue Chain) {
3820 int Mask[] = { Start, -1, Start + 1, -1 };
3821 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3822 if (Chain) {
3823 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3824 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3825 }
3826 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3827}
3828
3829// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3830// producing a result of type VT. If Chain is nonnull, return the strict form.
3831SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3832 const SDLoc &DL, EVT VT,
3833 SDValue CmpOp0,
3834 SDValue CmpOp1,
3835 SDValue Chain) const {
3836 // There is no hardware support for v4f32 (unless we have the vector
3837 // enhancements facility 1), so extend the vector into two v2f64s
3838 // and compare those.
3839 if (CmpOp0.getValueType() == MVT::v4f32 &&
3840 !Subtarget.hasVectorEnhancements1()) {
3841 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3842 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3843 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3844 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3845 if (Chain) {
3846 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3847 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3848 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3849 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3850 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3851 H1.getValue(R: 1), L1.getValue(R: 1),
3852 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3853 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3854 SDValue Ops[2] = { Res, NewChain };
3855 return DAG.getMergeValues(Ops, dl: DL);
3856 }
3857 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3858 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3859 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3860 }
3861 if (Chain) {
3862 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3863 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3864 }
3865 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3866}
3867
3868// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3869// an integer mask of type VT. If Chain is nonnull, we have a strict
3870// floating-point comparison. If in addition IsSignaling is true, we have
3871// a strict signaling floating-point comparison.
3872SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3873 const SDLoc &DL, EVT VT,
3874 ISD::CondCode CC,
3875 SDValue CmpOp0,
3876 SDValue CmpOp1,
3877 SDValue Chain,
3878 bool IsSignaling) const {
3879 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3880 assert (!Chain || IsFP);
3881 assert (!IsSignaling || Chain);
3882 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3883 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3884 bool Invert = false;
3885 SDValue Cmp;
3886 switch (CC) {
3887 // Handle tests for order using (or (ogt y x) (oge x y)).
3888 case ISD::SETUO:
3889 Invert = true;
3890 [[fallthrough]];
3891 case ISD::SETO: {
3892 assert(IsFP && "Unexpected integer comparison");
3893 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3894 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3895 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3896 DL, VT, CmpOp0, CmpOp1, Chain);
3897 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3898 if (Chain)
3899 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3900 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3901 break;
3902 }
3903
3904 // Handle <> tests using (or (ogt y x) (ogt x y)).
3905 case ISD::SETUEQ:
3906 Invert = true;
3907 [[fallthrough]];
3908 case ISD::SETONE: {
3909 assert(IsFP && "Unexpected integer comparison");
3910 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3911 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3912 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3913 DL, VT, CmpOp0, CmpOp1, Chain);
3914 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
3915 if (Chain)
3916 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3917 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
3918 break;
3919 }
3920
3921 // Otherwise a single comparison is enough. It doesn't really
3922 // matter whether we try the inversion or the swap first, since
3923 // there are no cases where both work.
3924 default:
3925 // Optimize sign-bit comparisons to signed compares.
3926 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
3927 ISD::isConstantSplatVectorAllZeros(N: CmpOp1.getNode())) {
3928 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3929 APInt Mask;
3930 if (CmpOp0.getOpcode() == ISD::AND
3931 && ISD::isConstantSplatVector(N: CmpOp0.getOperand(i: 1).getNode(), SplatValue&: Mask)
3932 && Mask == APInt::getSignMask(BitWidth: EltSize)) {
3933 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3934 CmpOp0 = CmpOp0.getOperand(i: 0);
3935 }
3936 }
3937 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3938 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3939 else {
3940 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3941 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3942 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3943 else
3944 llvm_unreachable("Unhandled comparison");
3945 }
3946 if (Chain)
3947 Chain = Cmp.getValue(R: 1);
3948 break;
3949 }
3950 if (Invert) {
3951 SDValue Mask =
3952 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
3953 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
3954 }
3955 if (Chain && Chain.getNode() != Cmp.getNode()) {
3956 SDValue Ops[2] = { Cmp, Chain };
3957 Cmp = DAG.getMergeValues(Ops, dl: DL);
3958 }
3959 return Cmp;
3960}
3961
3962SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3963 SelectionDAG &DAG) const {
3964 SDValue CmpOp0 = Op.getOperand(i: 0);
3965 SDValue CmpOp1 = Op.getOperand(i: 1);
3966 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3967 SDLoc DL(Op);
3968 EVT VT = Op.getValueType();
3969 if (VT.isVector())
3970 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3971
3972 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3973 SDValue CCReg = emitCmp(DAG, DL, C);
3974 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3975}
3976
3977SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
3978 SelectionDAG &DAG,
3979 bool IsSignaling) const {
3980 SDValue Chain = Op.getOperand(i: 0);
3981 SDValue CmpOp0 = Op.getOperand(i: 1);
3982 SDValue CmpOp1 = Op.getOperand(i: 2);
3983 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
3984 SDLoc DL(Op);
3985 EVT VT = Op.getNode()->getValueType(ResNo: 0);
3986 if (VT.isVector()) {
3987 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3988 Chain, IsSignaling);
3989 return Res.getValue(R: Op.getResNo());
3990 }
3991
3992 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
3993 SDValue CCReg = emitCmp(DAG, DL, C);
3994 CCReg->setFlags(Op->getFlags());
3995 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3996 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
3997 return DAG.getMergeValues(Ops, dl: DL);
3998}
3999
4000SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4001 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
4002 SDValue CmpOp0 = Op.getOperand(i: 2);
4003 SDValue CmpOp1 = Op.getOperand(i: 3);
4004 SDValue Dest = Op.getOperand(i: 4);
4005 SDLoc DL(Op);
4006
4007 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4008 SDValue CCReg = emitCmp(DAG, DL, C);
4009 return DAG.getNode(
4010 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
4011 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4012 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
4013}
4014
4015// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
4016// allowing Pos and Neg to be wider than CmpOp.
4017static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
4018 return (Neg.getOpcode() == ISD::SUB &&
4019 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
4020 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
4021 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
4022 Pos.getOperand(i: 0) == CmpOp)));
4023}
4024
4025// Return the absolute or negative absolute of Op; IsNegative decides which.
4026static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
4027 bool IsNegative) {
4028 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
4029 if (IsNegative)
4030 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
4031 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
4032 return Op;
4033}
4034
4035static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
4036 Comparison C, SDValue TrueOp, SDValue FalseOp) {
4037 EVT VT = MVT::i128;
4038 unsigned Op;
4039
4040 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
4041 C.CCMask == SystemZ::CCMASK_CMP_GE ||
4042 C.CCMask == SystemZ::CCMASK_CMP_LE) {
4043 std::swap(a&: TrueOp, b&: FalseOp);
4044 C.CCMask ^= C.CCValid;
4045 }
4046 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
4047 std::swap(a&: C.Op0, b&: C.Op1);
4048 C.CCMask = SystemZ::CCMASK_CMP_GT;
4049 }
4050 switch (C.CCMask) {
4051 case SystemZ::CCMASK_CMP_EQ:
4052 Op = SystemZISD::VICMPE;
4053 break;
4054 case SystemZ::CCMASK_CMP_GT:
4055 if (C.ICmpType == SystemZICMP::UnsignedOnly)
4056 Op = SystemZISD::VICMPHL;
4057 else
4058 Op = SystemZISD::VICMPH;
4059 break;
4060 default:
4061 llvm_unreachable("Unhandled comparison");
4062 break;
4063 }
4064
4065 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
4066 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
4067 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
4068 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
4069}
4070
4071SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4072 SelectionDAG &DAG) const {
4073 SDValue CmpOp0 = Op.getOperand(i: 0);
4074 SDValue CmpOp1 = Op.getOperand(i: 1);
4075 SDValue TrueOp = Op.getOperand(i: 2);
4076 SDValue FalseOp = Op.getOperand(i: 3);
4077 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
4078 SDLoc DL(Op);
4079
4080 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4081 // legalizer, as it will be handled according to the type of the resulting
4082 // value. Extend them here if needed.
4083 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4084 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
4085 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
4086 }
4087
4088 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4089
4090 // Check for absolute and negative-absolute selections, including those
4091 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4092 // This check supplements the one in DAGCombiner.
4093 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4094 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4095 C.Op1.getOpcode() == ISD::Constant &&
4096 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
4097 C.Op1->getAsZExtVal() == 0) {
4098 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
4099 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
4100 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
4101 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
4102 }
4103
4104 if (Subtarget.hasVectorEnhancements3() &&
4105 C.Opcode == SystemZISD::ICMP &&
4106 C.Op0.getValueType() == MVT::i128 &&
4107 TrueOp.getValueType() == MVT::i128) {
4108 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4109 }
4110
4111 SDValue CCReg = emitCmp(DAG, DL, C);
4112 SDValue Ops[] = {TrueOp, FalseOp,
4113 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4114 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
4115
4116 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
4117}
4118
4119SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4120 SelectionDAG &DAG) const {
4121 SDLoc DL(Node);
4122 const GlobalValue *GV = Node->getGlobal();
4123 int64_t Offset = Node->getOffset();
4124 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4125 CodeModel::Model CM = DAG.getTarget().getCodeModel();
4126
4127 SDValue Result;
4128 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4129 if (isInt<32>(x: Offset)) {
4130 // Assign anchors at 1<<12 byte boundaries.
4131 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4132 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
4133 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4134
4135 // The offset can be folded into the address if it is aligned to a
4136 // halfword.
4137 Offset -= Anchor;
4138 if (Offset != 0 && (Offset & 1) == 0) {
4139 SDValue Full =
4140 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
4141 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
4142 Offset = 0;
4143 }
4144 } else {
4145 // Conservatively load a constant offset greater than 32 bits into a
4146 // register below.
4147 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
4148 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4149 }
4150 } else if (Subtarget.isTargetELF()) {
4151 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
4152 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4153 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
4154 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4155 } else if (Subtarget.isTargetzOS()) {
4156 Result = getADAEntry(DAG, GV, DL, PtrVT);
4157 } else
4158 llvm_unreachable("Unexpected Subtarget");
4159
4160 // If there was a non-zero offset that we didn't fold, create an explicit
4161 // addition for it.
4162 if (Offset != 0)
4163 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4164 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4165
4166 return Result;
4167}
4168
4169SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4170 SelectionDAG &DAG,
4171 unsigned Opcode,
4172 SDValue GOTOffset) const {
4173 SDLoc DL(Node);
4174 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4175 SDValue Chain = DAG.getEntryNode();
4176 SDValue Glue;
4177
4178 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4179 CallingConv::GHC)
4180 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4181
4182 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4183 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4184 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4185 Glue = Chain.getValue(R: 1);
4186 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4187 Glue = Chain.getValue(R: 1);
4188
4189 // The first call operand is the chain and the second is the TLS symbol.
4190 SmallVector<SDValue, 8> Ops;
4191 Ops.push_back(Elt: Chain);
4192 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4193 VT: Node->getValueType(ResNo: 0),
4194 offset: 0, TargetFlags: 0));
4195
4196 // Add argument registers to the end of the list so that they are
4197 // known live into the call.
4198 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4199 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4200
4201 // Add a register mask operand representing the call-preserved registers.
4202 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4203 const uint32_t *Mask =
4204 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4205 assert(Mask && "Missing call preserved mask for calling convention");
4206 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4207
4208 // Glue the call to the argument copies.
4209 Ops.push_back(Elt: Glue);
4210
4211 // Emit the call.
4212 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4213 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4214 Glue = Chain.getValue(R: 1);
4215
4216 // Copy the return value from %r2.
4217 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4218}
4219
4220SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4221 SelectionDAG &DAG) const {
4222 SDValue Chain = DAG.getEntryNode();
4223 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4224
4225 // The high part of the thread pointer is in access register 0.
4226 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4227 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4228
4229 // The low part of the thread pointer is in access register 1.
4230 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4231 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4232
4233 // Merge them into a single 64-bit address.
4234 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4235 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4236 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4237}
4238
4239SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4240 SelectionDAG &DAG) const {
4241 if (DAG.getTarget().useEmulatedTLS())
4242 return LowerToTLSEmulatedModel(GA: Node, DAG);
4243 SDLoc DL(Node);
4244 const GlobalValue *GV = Node->getGlobal();
4245 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4246 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4247
4248 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4249 CallingConv::GHC)
4250 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4251
4252 SDValue TP = lowerThreadPointer(DL, DAG);
4253
4254 // Get the offset of GA from the thread pointer, based on the TLS model.
4255 SDValue Offset;
4256 switch (model) {
4257 case TLSModel::GeneralDynamic: {
4258 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4259 SystemZConstantPoolValue *CPV =
4260 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4261
4262 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4263 Offset = DAG.getLoad(
4264 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4265 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4266
4267 // Call __tls_get_offset to retrieve the offset.
4268 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4269 break;
4270 }
4271
4272 case TLSModel::LocalDynamic: {
4273 // Load the GOT offset of the module ID.
4274 SystemZConstantPoolValue *CPV =
4275 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4276
4277 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4278 Offset = DAG.getLoad(
4279 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4280 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4281
4282 // Call __tls_get_offset to retrieve the module base offset.
4283 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4284
4285 // Note: The SystemZLDCleanupPass will remove redundant computations
4286 // of the module base offset. Count total number of local-dynamic
4287 // accesses to trigger execution of that pass.
4288 SystemZMachineFunctionInfo* MFI =
4289 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4290 MFI->incNumLocalDynamicTLSAccesses();
4291
4292 // Add the per-symbol offset.
4293 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4294
4295 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4296 DTPOffset = DAG.getLoad(
4297 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4298 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4299
4300 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4301 break;
4302 }
4303
4304 case TLSModel::InitialExec: {
4305 // Load the offset from the GOT.
4306 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4307 TargetFlags: SystemZII::MO_INDNTPOFF);
4308 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4309 Offset =
4310 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4311 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4312 break;
4313 }
4314
4315 case TLSModel::LocalExec: {
4316 // Force the offset into the constant pool and load it from there.
4317 SystemZConstantPoolValue *CPV =
4318 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4319
4320 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4321 Offset = DAG.getLoad(
4322 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4323 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4324 break;
4325 }
4326 }
4327
4328 // Add the base and offset together.
4329 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4330}
4331
4332SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4333 SelectionDAG &DAG) const {
4334 SDLoc DL(Node);
4335 const BlockAddress *BA = Node->getBlockAddress();
4336 int64_t Offset = Node->getOffset();
4337 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4338
4339 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4340 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4341 return Result;
4342}
4343
4344SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4345 SelectionDAG &DAG) const {
4346 SDLoc DL(JT);
4347 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4348 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4349
4350 // Use LARL to load the address of the table.
4351 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4352}
4353
4354SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4355 SelectionDAG &DAG) const {
4356 SDLoc DL(CP);
4357 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4358
4359 SDValue Result;
4360 if (CP->isMachineConstantPoolEntry())
4361 Result =
4362 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4363 else
4364 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4365 Offset: CP->getOffset());
4366
4367 // Use LARL to load the address of the constant pool entry.
4368 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4369}
4370
4371SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4372 SelectionDAG &DAG) const {
4373 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4374 MachineFunction &MF = DAG.getMachineFunction();
4375 MachineFrameInfo &MFI = MF.getFrameInfo();
4376 MFI.setFrameAddressIsTaken(true);
4377
4378 SDLoc DL(Op);
4379 unsigned Depth = Op.getConstantOperandVal(i: 0);
4380 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4381
4382 // By definition, the frame address is the address of the back chain. (In
4383 // the case of packed stack without backchain, return the address where the
4384 // backchain would have been stored. This will either be an unused space or
4385 // contain a saved register).
4386 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4387 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4388
4389 if (Depth > 0) {
4390 // FIXME The frontend should detect this case.
4391 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4392 report_fatal_error(reason: "Unsupported stack frame traversal count");
4393
4394 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4395 while (Depth--) {
4396 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4397 PtrInfo: MachinePointerInfo());
4398 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4399 }
4400 }
4401
4402 return BackChain;
4403}
4404
4405SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4406 SelectionDAG &DAG) const {
4407 MachineFunction &MF = DAG.getMachineFunction();
4408 MachineFrameInfo &MFI = MF.getFrameInfo();
4409 MFI.setReturnAddressIsTaken(true);
4410
4411 SDLoc DL(Op);
4412 unsigned Depth = Op.getConstantOperandVal(i: 0);
4413 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4414
4415 if (Depth > 0) {
4416 // FIXME The frontend should detect this case.
4417 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4418 report_fatal_error(reason: "Unsupported stack frame traversal count");
4419
4420 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4421 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4422 int Offset = TFL->getReturnAddressOffset(MF);
4423 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4424 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4425 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4426 PtrInfo: MachinePointerInfo());
4427 }
4428
4429 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4430 // implicit live-in.
4431 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4432 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4433 RC: &SystemZ::GR64BitRegClass);
4434 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4435}
4436
4437SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4438 SelectionDAG &DAG) const {
4439 SDLoc DL(Op);
4440 SDValue In = Op.getOperand(i: 0);
4441 EVT InVT = In.getValueType();
4442 EVT ResVT = Op.getValueType();
4443
4444 // Convert loads directly. This is normally done by DAGCombiner,
4445 // but we need this case for bitcasts that are created during lowering
4446 // and which are then lowered themselves.
4447 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4448 if (ISD::isNormalLoad(N: LoadN)) {
4449 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4450 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4451 // Update the chain uses.
4452 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4453 return NewLoad;
4454 }
4455
4456 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4457 SDValue In64;
4458 if (Subtarget.hasHighWord()) {
4459 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4460 VT: MVT::i64);
4461 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4462 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4463 } else {
4464 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4465 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4466 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4467 }
4468 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4469 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4470 DL, VT: MVT::f32, Operand: Out64);
4471 }
4472 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4473 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4474 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4475 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4476 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4477 if (Subtarget.hasHighWord())
4478 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4479 VT: MVT::i32, Operand: Out64);
4480 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4481 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4482 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4483 }
4484 llvm_unreachable("Unexpected bitcast combination");
4485}
4486
4487SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4488 SelectionDAG &DAG) const {
4489
4490 if (Subtarget.isTargetXPLINK64())
4491 return lowerVASTART_XPLINK(Op, DAG);
4492 else
4493 return lowerVASTART_ELF(Op, DAG);
4494}
4495
4496SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4497 SelectionDAG &DAG) const {
4498 MachineFunction &MF = DAG.getMachineFunction();
4499 SystemZMachineFunctionInfo *FuncInfo =
4500 MF.getInfo<SystemZMachineFunctionInfo>();
4501
4502 SDLoc DL(Op);
4503
4504 // vastart just stores the address of the VarArgsFrameIndex slot into the
4505 // memory location argument.
4506 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4507 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4508 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4509 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4510 PtrInfo: MachinePointerInfo(SV));
4511}
4512
4513SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4514 SelectionDAG &DAG) const {
4515 MachineFunction &MF = DAG.getMachineFunction();
4516 SystemZMachineFunctionInfo *FuncInfo =
4517 MF.getInfo<SystemZMachineFunctionInfo>();
4518 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4519
4520 SDValue Chain = Op.getOperand(i: 0);
4521 SDValue Addr = Op.getOperand(i: 1);
4522 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4523 SDLoc DL(Op);
4524
4525 // The initial values of each field.
4526 const unsigned NumFields = 4;
4527 SDValue Fields[NumFields] = {
4528 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4529 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4530 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4531 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4532 };
4533
4534 // Store each field into its respective slot.
4535 SDValue MemOps[NumFields];
4536 unsigned Offset = 0;
4537 for (unsigned I = 0; I < NumFields; ++I) {
4538 SDValue FieldAddr = Addr;
4539 if (Offset != 0)
4540 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4541 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4542 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4543 PtrInfo: MachinePointerInfo(SV, Offset));
4544 Offset += 8;
4545 }
4546 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4547}
4548
4549SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4550 SelectionDAG &DAG) const {
4551 SDValue Chain = Op.getOperand(i: 0);
4552 SDValue DstPtr = Op.getOperand(i: 1);
4553 SDValue SrcPtr = Op.getOperand(i: 2);
4554 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4555 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4556 SDLoc DL(Op);
4557
4558 uint32_t Sz =
4559 Subtarget.isTargetXPLINK64() ? DAG.getDataLayout().getPointerSize(AS: 0) : 32;
4560 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4561 DstAlign: Align(8), SrcAlign: Align(8), /*isVolatile*/ isVol: false,
4562 /*AlwaysInline*/ false,
4563 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4564 SrcPtrInfo: MachinePointerInfo(SrcSV));
4565}
4566
4567SDValue
4568SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4569 SelectionDAG &DAG) const {
4570 if (Subtarget.isTargetXPLINK64())
4571 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4572 else
4573 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4574}
4575
4576SDValue
4577SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4578 SelectionDAG &DAG) const {
4579 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4580 MachineFunction &MF = DAG.getMachineFunction();
4581 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4582 SDValue Chain = Op.getOperand(i: 0);
4583 SDValue Size = Op.getOperand(i: 1);
4584 SDValue Align = Op.getOperand(i: 2);
4585 SDLoc DL(Op);
4586
4587 // If user has set the no alignment function attribute, ignore
4588 // alloca alignments.
4589 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4590
4591 uint64_t StackAlign = TFI->getStackAlignment();
4592 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4593 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4594
4595 SDValue NeededSpace = Size;
4596
4597 // Add extra space for alignment if needed.
4598 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4599 if (ExtraAlignSpace)
4600 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4601 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4602
4603 bool IsSigned = false;
4604 bool DoesNotReturn = false;
4605 bool IsReturnValueUsed = false;
4606 EVT VT = Op.getValueType();
4607 SDValue AllocaCall =
4608 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4609 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4610 IsReturnValueUsed)
4611 .first;
4612
4613 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4614 // to end of call in order to ensure it isn't broken up from the call
4615 // sequence.
4616 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4617 Register SPReg = Regs.getStackPointerRegister();
4618 Chain = AllocaCall.getValue(R: 1);
4619 SDValue Glue = AllocaCall.getValue(R: 2);
4620 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4621 Chain = NewSPRegNode.getValue(R: 1);
4622
4623 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4624 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4625 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4626
4627 // Dynamically realign if needed.
4628 if (ExtraAlignSpace) {
4629 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4630 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4631 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4632 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4633 }
4634
4635 SDValue Ops[2] = {Result, Chain};
4636 return DAG.getMergeValues(Ops, dl: DL);
4637}
4638
4639SDValue
4640SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4641 SelectionDAG &DAG) const {
4642 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4643 MachineFunction &MF = DAG.getMachineFunction();
4644 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4645 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4646
4647 SDValue Chain = Op.getOperand(i: 0);
4648 SDValue Size = Op.getOperand(i: 1);
4649 SDValue Align = Op.getOperand(i: 2);
4650 SDLoc DL(Op);
4651
4652 // If user has set the no alignment function attribute, ignore
4653 // alloca alignments.
4654 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4655
4656 uint64_t StackAlign = TFI->getStackAlignment();
4657 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4658 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4659
4660 Register SPReg = getStackPointerRegisterToSaveRestore();
4661 SDValue NeededSpace = Size;
4662
4663 // Get a reference to the stack pointer.
4664 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4665
4666 // If we need a backchain, save it now.
4667 SDValue Backchain;
4668 if (StoreBackchain)
4669 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4670 PtrInfo: MachinePointerInfo());
4671
4672 // Add extra space for alignment if needed.
4673 if (ExtraAlignSpace)
4674 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4675 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4676
4677 // Get the new stack pointer value.
4678 SDValue NewSP;
4679 if (hasInlineStackProbe(MF)) {
4680 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4681 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4682 Chain = NewSP.getValue(R: 1);
4683 }
4684 else {
4685 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4686 // Copy the new stack pointer back.
4687 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4688 }
4689
4690 // The allocated data lives above the 160 bytes allocated for the standard
4691 // frame, plus any outgoing stack arguments. We don't know how much that
4692 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4693 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4694 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4695
4696 // Dynamically realign if needed.
4697 if (RequiredAlign > StackAlign) {
4698 Result =
4699 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4700 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4701 Result =
4702 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4703 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4704 }
4705
4706 if (StoreBackchain)
4707 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4708 PtrInfo: MachinePointerInfo());
4709
4710 SDValue Ops[2] = { Result, Chain };
4711 return DAG.getMergeValues(Ops, dl: DL);
4712}
4713
4714SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4715 SDValue Op, SelectionDAG &DAG) const {
4716 SDLoc DL(Op);
4717
4718 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4719}
4720
4721SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4722 SelectionDAG &DAG,
4723 unsigned Opcode) const {
4724 EVT VT = Op.getValueType();
4725 SDLoc DL(Op);
4726 SDValue Even, Odd;
4727
4728 // This custom expander is only used on z17 and later for 64-bit types.
4729 assert(!is32Bit(VT));
4730 assert(Subtarget.hasMiscellaneousExtensions2());
4731
4732 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4733 // the high result in the even register. Return the latter.
4734 lowerGR128Binary(DAG, DL, VT, Opcode,
4735 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4736 return Even;
4737}
4738
4739SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4740 SelectionDAG &DAG) const {
4741 EVT VT = Op.getValueType();
4742 SDLoc DL(Op);
4743 SDValue Ops[2];
4744 if (is32Bit(VT))
4745 // Just do a normal 64-bit multiplication and extract the results.
4746 // We define this so that it can be used for constant division.
4747 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4748 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4749 else if (Subtarget.hasMiscellaneousExtensions2())
4750 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4751 // the high result in the even register. ISD::SMUL_LOHI is defined to
4752 // return the low half first, so the results are in reverse order.
4753 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4754 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4755 else {
4756 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4757 //
4758 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4759 //
4760 // but using the fact that the upper halves are either all zeros
4761 // or all ones:
4762 //
4763 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4764 //
4765 // and grouping the right terms together since they are quicker than the
4766 // multiplication:
4767 //
4768 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4769 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4770 SDValue LL = Op.getOperand(i: 0);
4771 SDValue RL = Op.getOperand(i: 1);
4772 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4773 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4774 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4775 // the high result in the even register. ISD::SMUL_LOHI is defined to
4776 // return the low half first, so the results are in reverse order.
4777 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4778 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4779 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4780 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4781 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4782 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4783 }
4784 return DAG.getMergeValues(Ops, dl: DL);
4785}
4786
4787SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4788 SelectionDAG &DAG) const {
4789 EVT VT = Op.getValueType();
4790 SDLoc DL(Op);
4791 SDValue Ops[2];
4792 if (is32Bit(VT))
4793 // Just do a normal 64-bit multiplication and extract the results.
4794 // We define this so that it can be used for constant division.
4795 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4796 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4797 else
4798 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4799 // the high result in the even register. ISD::UMUL_LOHI is defined to
4800 // return the low half first, so the results are in reverse order.
4801 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4802 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4803 return DAG.getMergeValues(Ops, dl: DL);
4804}
4805
4806SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4807 SelectionDAG &DAG) const {
4808 SDValue Op0 = Op.getOperand(i: 0);
4809 SDValue Op1 = Op.getOperand(i: 1);
4810 EVT VT = Op.getValueType();
4811 SDLoc DL(Op);
4812
4813 // We use DSGF for 32-bit division. This means the first operand must
4814 // always be 64-bit, and the second operand should be 32-bit whenever
4815 // that is possible, to improve performance.
4816 if (is32Bit(VT))
4817 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4818 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4819 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4820
4821 // DSG(F) returns the remainder in the even register and the
4822 // quotient in the odd register.
4823 SDValue Ops[2];
4824 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4825 return DAG.getMergeValues(Ops, dl: DL);
4826}
4827
4828SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4829 SelectionDAG &DAG) const {
4830 EVT VT = Op.getValueType();
4831 SDLoc DL(Op);
4832
4833 // DL(G) returns the remainder in the even register and the
4834 // quotient in the odd register.
4835 SDValue Ops[2];
4836 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4837 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4838 return DAG.getMergeValues(Ops, dl: DL);
4839}
4840
4841SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4842 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4843
4844 // Get the known-zero masks for each operand.
4845 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4846 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4847 DAG.computeKnownBits(Op: Ops[1])};
4848
4849 // See if the upper 32 bits of one operand and the lower 32 bits of the
4850 // other are known zero. They are the low and high operands respectively.
4851 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4852 Known[1].Zero.getZExtValue() };
4853 unsigned High, Low;
4854 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4855 High = 1, Low = 0;
4856 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4857 High = 0, Low = 1;
4858 else
4859 return Op;
4860
4861 SDValue LowOp = Ops[Low];
4862 SDValue HighOp = Ops[High];
4863
4864 // If the high part is a constant, we're better off using IILH.
4865 if (HighOp.getOpcode() == ISD::Constant)
4866 return Op;
4867
4868 // If the low part is a constant that is outside the range of LHI,
4869 // then we're better off using IILF.
4870 if (LowOp.getOpcode() == ISD::Constant) {
4871 int64_t Value = int32_t(LowOp->getAsZExtVal());
4872 if (!isInt<16>(x: Value))
4873 return Op;
4874 }
4875
4876 // Check whether the high part is an AND that doesn't change the
4877 // high 32 bits and just masks out low bits. We can skip it if so.
4878 if (HighOp.getOpcode() == ISD::AND &&
4879 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4880 SDValue HighOp0 = HighOp.getOperand(i: 0);
4881 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4882 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4883 HighOp = HighOp0;
4884 }
4885
4886 // Take advantage of the fact that all GR32 operations only change the
4887 // low 32 bits by truncating Low to an i32 and inserting it directly
4888 // using a subreg. The interesting cases are those where the truncation
4889 // can be folded.
4890 SDLoc DL(Op);
4891 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4892 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4893 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4894}
4895
4896// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4897SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4898 SelectionDAG &DAG) const {
4899 SDNode *N = Op.getNode();
4900 SDValue LHS = N->getOperand(Num: 0);
4901 SDValue RHS = N->getOperand(Num: 1);
4902 SDLoc DL(N);
4903
4904 if (N->getValueType(ResNo: 0) == MVT::i128) {
4905 unsigned BaseOp = 0;
4906 unsigned FlagOp = 0;
4907 bool IsBorrow = false;
4908 switch (Op.getOpcode()) {
4909 default: llvm_unreachable("Unknown instruction!");
4910 case ISD::UADDO:
4911 BaseOp = ISD::ADD;
4912 FlagOp = SystemZISD::VACC;
4913 break;
4914 case ISD::USUBO:
4915 BaseOp = ISD::SUB;
4916 FlagOp = SystemZISD::VSCBI;
4917 IsBorrow = true;
4918 break;
4919 }
4920 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4921 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4922 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4923 N2: DAG.getValueType(MVT::i1));
4924 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4925 if (IsBorrow)
4926 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4927 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4928 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4929 }
4930
4931 unsigned BaseOp = 0;
4932 unsigned CCValid = 0;
4933 unsigned CCMask = 0;
4934
4935 switch (Op.getOpcode()) {
4936 default: llvm_unreachable("Unknown instruction!");
4937 case ISD::SADDO:
4938 BaseOp = SystemZISD::SADDO;
4939 CCValid = SystemZ::CCMASK_ARITH;
4940 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4941 break;
4942 case ISD::SSUBO:
4943 BaseOp = SystemZISD::SSUBO;
4944 CCValid = SystemZ::CCMASK_ARITH;
4945 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4946 break;
4947 case ISD::UADDO:
4948 BaseOp = SystemZISD::UADDO;
4949 CCValid = SystemZ::CCMASK_LOGICAL;
4950 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4951 break;
4952 case ISD::USUBO:
4953 BaseOp = SystemZISD::USUBO;
4954 CCValid = SystemZ::CCMASK_LOGICAL;
4955 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4956 break;
4957 }
4958
4959 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
4960 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
4961
4962 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4963 if (N->getValueType(ResNo: 1) == MVT::i1)
4964 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4965
4966 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4967}
4968
4969static bool isAddCarryChain(SDValue Carry) {
4970 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
4971 Carry->getValueType(ResNo: 0) != MVT::i128)
4972 Carry = Carry.getOperand(i: 2);
4973 return Carry.getOpcode() == ISD::UADDO &&
4974 Carry->getValueType(ResNo: 0) != MVT::i128;
4975}
4976
4977static bool isSubBorrowChain(SDValue Carry) {
4978 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
4979 Carry->getValueType(ResNo: 0) != MVT::i128)
4980 Carry = Carry.getOperand(i: 2);
4981 return Carry.getOpcode() == ISD::USUBO &&
4982 Carry->getValueType(ResNo: 0) != MVT::i128;
4983}
4984
4985// Lower UADDO_CARRY/USUBO_CARRY nodes.
4986SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
4987 SelectionDAG &DAG) const {
4988
4989 SDNode *N = Op.getNode();
4990 MVT VT = N->getSimpleValueType(ResNo: 0);
4991
4992 // Let legalize expand this if it isn't a legal type yet.
4993 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4994 return SDValue();
4995
4996 SDValue LHS = N->getOperand(Num: 0);
4997 SDValue RHS = N->getOperand(Num: 1);
4998 SDValue Carry = Op.getOperand(i: 2);
4999 SDLoc DL(N);
5000
5001 if (VT == MVT::i128) {
5002 unsigned BaseOp = 0;
5003 unsigned FlagOp = 0;
5004 bool IsBorrow = false;
5005 switch (Op.getOpcode()) {
5006 default: llvm_unreachable("Unknown instruction!");
5007 case ISD::UADDO_CARRY:
5008 BaseOp = SystemZISD::VAC;
5009 FlagOp = SystemZISD::VACCC;
5010 break;
5011 case ISD::USUBO_CARRY:
5012 BaseOp = SystemZISD::VSBI;
5013 FlagOp = SystemZISD::VSBCBI;
5014 IsBorrow = true;
5015 break;
5016 }
5017 if (IsBorrow)
5018 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
5019 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
5020 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
5021 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5022 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5023 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
5024 N2: DAG.getValueType(MVT::i1));
5025 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
5026 if (IsBorrow)
5027 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
5028 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
5029 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
5030 }
5031
5032 unsigned BaseOp = 0;
5033 unsigned CCValid = 0;
5034 unsigned CCMask = 0;
5035
5036 switch (Op.getOpcode()) {
5037 default: llvm_unreachable("Unknown instruction!");
5038 case ISD::UADDO_CARRY:
5039 if (!isAddCarryChain(Carry))
5040 return SDValue();
5041
5042 BaseOp = SystemZISD::ADDCARRY;
5043 CCValid = SystemZ::CCMASK_LOGICAL;
5044 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
5045 break;
5046 case ISD::USUBO_CARRY:
5047 if (!isSubBorrowChain(Carry))
5048 return SDValue();
5049
5050 BaseOp = SystemZISD::SUBCARRY;
5051 CCValid = SystemZ::CCMASK_LOGICAL;
5052 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
5053 break;
5054 }
5055
5056 // Set the condition code from the carry flag.
5057 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
5058 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
5059 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
5060
5061 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
5062 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
5063
5064 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
5065 if (N->getValueType(ResNo: 1) == MVT::i1)
5066 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5067
5068 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5069}
5070
5071SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5072 SelectionDAG &DAG) const {
5073 EVT VT = Op.getValueType();
5074 SDLoc DL(Op);
5075 Op = Op.getOperand(i: 0);
5076
5077 if (VT.getScalarSizeInBits() == 128) {
5078 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
5079 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
5080 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
5081 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
5082 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5083 return Op;
5084 }
5085
5086 // Handle vector types via VPOPCT.
5087 if (VT.isVector()) {
5088 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
5089 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
5090 switch (VT.getScalarSizeInBits()) {
5091 case 8:
5092 break;
5093 case 16: {
5094 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
5095 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
5096 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5097 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5098 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5099 break;
5100 }
5101 case 32: {
5102 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5103 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5104 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5105 break;
5106 }
5107 case 64: {
5108 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5109 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5110 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
5111 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v4i32, Operand: Tmp);
5112 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5113 break;
5114 }
5115 default:
5116 llvm_unreachable("Unexpected type");
5117 }
5118 return Op;
5119 }
5120
5121 // Get the known-zero mask for the operand.
5122 KnownBits Known = DAG.computeKnownBits(Op);
5123 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5124 if (NumSignificantBits == 0)
5125 return DAG.getConstant(Val: 0, DL, VT);
5126
5127 // Skip known-zero high parts of the operand.
5128 int64_t OrigBitSize = VT.getSizeInBits();
5129 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
5130 BitSize = std::min(a: BitSize, b: OrigBitSize);
5131
5132 // The POPCNT instruction counts the number of bits in each byte.
5133 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
5134 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
5135 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
5136
5137 // Add up per-byte counts in a binary tree. All bits of Op at
5138 // position larger than BitSize remain zero throughout.
5139 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5140 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
5141 if (BitSize != OrigBitSize)
5142 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
5143 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
5144 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5145 }
5146
5147 // Extract overall result from high byte.
5148 if (BitSize > 8)
5149 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
5150 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
5151
5152 return Op;
5153}
5154
5155SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5156 SelectionDAG &DAG) const {
5157 SDLoc DL(Op);
5158 AtomicOrdering FenceOrdering =
5159 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5160 SyncScope::ID FenceSSID =
5161 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5162
5163 // The only fence that needs an instruction is a sequentially-consistent
5164 // cross-thread fence.
5165 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5166 FenceSSID == SyncScope::System) {
5167 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5168 Op1: Op.getOperand(i: 0)),
5169 0);
5170 }
5171
5172 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5173 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5174}
5175
5176SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5177 SelectionDAG &DAG) const {
5178 EVT RegVT = Op.getValueType();
5179 if (RegVT.getSizeInBits() == 128)
5180 return lowerATOMIC_LDST_I128(Op, DAG);
5181 return lowerLoadF16(Op, DAG);
5182}
5183
5184SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5185 SelectionDAG &DAG) const {
5186 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5187 if (Node->getMemoryVT().getSizeInBits() == 128)
5188 return lowerATOMIC_LDST_I128(Op, DAG);
5189 return lowerStoreF16(Op, DAG);
5190}
5191
5192SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5193 SelectionDAG &DAG) const {
5194 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5195 assert(
5196 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5197 "Only custom lowering i128 or f128.");
5198 // Use same code to handle both legal and non-legal i128 types.
5199 SmallVector<SDValue, 2> Results;
5200 LowerOperationWrapper(N: Node, Results, DAG);
5201 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5202}
5203
5204// Prepare for a Compare And Swap for a subword operation. This needs to be
5205// done in memory with 4 bytes at natural alignment.
5206static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5207 SDValue &AlignedAddr, SDValue &BitShift,
5208 SDValue &NegBitShift) {
5209 EVT PtrVT = Addr.getValueType();
5210 EVT WideVT = MVT::i32;
5211
5212 // Get the address of the containing word.
5213 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5214 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5215
5216 // Get the number of bits that the word must be rotated left in order
5217 // to bring the field to the top bits of a GR32.
5218 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5219 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5220 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5221
5222 // Get the complementing shift amount, for rotating a field in the top
5223 // bits back to its proper position.
5224 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5225 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5226
5227}
5228
5229// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
5230// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
5231SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
5232 SelectionDAG &DAG,
5233 unsigned Opcode) const {
5234 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5235
5236 // 32-bit operations need no special handling.
5237 EVT NarrowVT = Node->getMemoryVT();
5238 EVT WideVT = MVT::i32;
5239 if (NarrowVT == WideVT)
5240 return Op;
5241
5242 int64_t BitSize = NarrowVT.getSizeInBits();
5243 SDValue ChainIn = Node->getChain();
5244 SDValue Addr = Node->getBasePtr();
5245 SDValue Src2 = Node->getVal();
5246 MachineMemOperand *MMO = Node->getMemOperand();
5247 SDLoc DL(Node);
5248
5249 // Convert atomic subtracts of constants into additions.
5250 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5251 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5252 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5253 Src2 = DAG.getSignedConstant(Val: -Const->getSExtValue(), DL,
5254 VT: Src2.getValueType());
5255 }
5256
5257 SDValue AlignedAddr, BitShift, NegBitShift;
5258 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5259
5260 // Extend the source operand to 32 bits and prepare it for the inner loop.
5261 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5262 // operations require the source to be shifted in advance. (This shift
5263 // can be folded if the source is constant.) For AND and NAND, the lower
5264 // bits must be set, while for other opcodes they should be left clear.
5265 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5266 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5267 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5268 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5269 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5270 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5271 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5272
5273 // Construct the ATOMIC_LOADW_* node.
5274 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5275 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5276 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5277 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5278 MemVT: NarrowVT, MMO);
5279
5280 // Rotate the result of the final CS so that the field is in the lower
5281 // bits of a GR32, then truncate it.
5282 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5283 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5284 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5285
5286 SDValue RetOps[2] = { Result, AtomicOp.getValue(R: 1) };
5287 return DAG.getMergeValues(Ops: RetOps, dl: DL);
5288}
5289
5290// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
5291// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
5292SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5293 SelectionDAG &DAG) const {
5294 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5295 EVT MemVT = Node->getMemoryVT();
5296 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
5297 // A full-width operation: negate and use LAA(G).
5298 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5299 assert(Subtarget.hasInterlockedAccess1() &&
5300 "Should have been expanded by AtomicExpand pass.");
5301 SDValue Src2 = Node->getVal();
5302 SDLoc DL(Src2);
5303 SDValue NegSrc2 =
5304 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5305 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT,
5306 Chain: Node->getChain(), Ptr: Node->getBasePtr(), Val: NegSrc2,
5307 MMO: Node->getMemOperand());
5308 }
5309
5310 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
5311}
5312
5313// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5314SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5315 SelectionDAG &DAG) const {
5316 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5317 SDValue ChainIn = Node->getOperand(Num: 0);
5318 SDValue Addr = Node->getOperand(Num: 1);
5319 SDValue CmpVal = Node->getOperand(Num: 2);
5320 SDValue SwapVal = Node->getOperand(Num: 3);
5321 MachineMemOperand *MMO = Node->getMemOperand();
5322 SDLoc DL(Node);
5323
5324 if (Node->getMemoryVT() == MVT::i128) {
5325 // Use same code to handle both legal and non-legal i128 types.
5326 SmallVector<SDValue, 3> Results;
5327 LowerOperationWrapper(N: Node, Results, DAG);
5328 return DAG.getMergeValues(Ops: Results, dl: DL);
5329 }
5330
5331 // We have native support for 32-bit and 64-bit compare and swap, but we
5332 // still need to expand extracting the "success" result from the CC.
5333 EVT NarrowVT = Node->getMemoryVT();
5334 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5335 if (NarrowVT == WideVT) {
5336 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5337 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5338 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5339 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5340 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5341 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5342
5343 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5344 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5345 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5346 return SDValue();
5347 }
5348
5349 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5350 // via a fullword ATOMIC_CMP_SWAPW operation.
5351 int64_t BitSize = NarrowVT.getSizeInBits();
5352
5353 SDValue AlignedAddr, BitShift, NegBitShift;
5354 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5355
5356 // Construct the ATOMIC_CMP_SWAPW node.
5357 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5358 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5359 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5360 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5361 VTList, Ops, MemVT: NarrowVT, MMO);
5362 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5363 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5364
5365 // emitAtomicCmpSwapW() will zero extend the result (original value).
5366 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5367 N2: DAG.getValueType(NarrowVT));
5368 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5369 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5370 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5371 return SDValue();
5372}
5373
5374MachineMemOperand::Flags
5375SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5376 // Because of how we convert atomic_load and atomic_store to normal loads and
5377 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5378 // since DAGCombine hasn't been updated to account for atomic, but non
5379 // volatile loads. (See D57601)
5380 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5381 if (SI->isAtomic())
5382 return MachineMemOperand::MOVolatile;
5383 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5384 if (LI->isAtomic())
5385 return MachineMemOperand::MOVolatile;
5386 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5387 if (AI->isAtomic())
5388 return MachineMemOperand::MOVolatile;
5389 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5390 if (AI->isAtomic())
5391 return MachineMemOperand::MOVolatile;
5392 return MachineMemOperand::MONone;
5393}
5394
5395SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5396 SelectionDAG &DAG) const {
5397 MachineFunction &MF = DAG.getMachineFunction();
5398 auto *Regs = Subtarget.getSpecialRegisters();
5399 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5400 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5401 "in GHC calling convention");
5402 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5403 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5404}
5405
5406SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5407 SelectionDAG &DAG) const {
5408 MachineFunction &MF = DAG.getMachineFunction();
5409 auto *Regs = Subtarget.getSpecialRegisters();
5410 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5411
5412 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5413 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5414 "in GHC calling convention");
5415
5416 SDValue Chain = Op.getOperand(i: 0);
5417 SDValue NewSP = Op.getOperand(i: 1);
5418 SDValue Backchain;
5419 SDLoc DL(Op);
5420
5421 if (StoreBackchain) {
5422 SDValue OldSP = DAG.getCopyFromReg(
5423 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5424 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5425 PtrInfo: MachinePointerInfo());
5426 }
5427
5428 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5429
5430 if (StoreBackchain)
5431 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5432 PtrInfo: MachinePointerInfo());
5433
5434 return Chain;
5435}
5436
5437SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5438 SelectionDAG &DAG) const {
5439 bool IsData = Op.getConstantOperandVal(i: 4);
5440 if (!IsData)
5441 // Just preserve the chain.
5442 return Op.getOperand(i: 0);
5443
5444 SDLoc DL(Op);
5445 bool IsWrite = Op.getConstantOperandVal(i: 2);
5446 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5447 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5448 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5449 Op.getOperand(i: 1)};
5450 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5451 VTList: Node->getVTList(), Ops,
5452 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5453}
5454
5455SDValue
5456SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5457 SelectionDAG &DAG) const {
5458 unsigned Opcode, CCValid;
5459 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5460 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5461 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5462 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5463 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5464 return SDValue();
5465 }
5466
5467 return SDValue();
5468}
5469
5470SDValue
5471SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5472 SelectionDAG &DAG) const {
5473 unsigned Opcode, CCValid;
5474 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5475 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5476 if (Op->getNumValues() == 1)
5477 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5478 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5479 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5480 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5481 }
5482
5483 unsigned Id = Op.getConstantOperandVal(i: 0);
5484 switch (Id) {
5485 case Intrinsic::thread_pointer:
5486 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5487
5488 case Intrinsic::s390_vpdi:
5489 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5490 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5491
5492 case Intrinsic::s390_vperm:
5493 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5494 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5495
5496 case Intrinsic::s390_vuphb:
5497 case Intrinsic::s390_vuphh:
5498 case Intrinsic::s390_vuphf:
5499 case Intrinsic::s390_vuphg:
5500 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5501 Operand: Op.getOperand(i: 1));
5502
5503 case Intrinsic::s390_vuplhb:
5504 case Intrinsic::s390_vuplhh:
5505 case Intrinsic::s390_vuplhf:
5506 case Intrinsic::s390_vuplhg:
5507 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5508 Operand: Op.getOperand(i: 1));
5509
5510 case Intrinsic::s390_vuplb:
5511 case Intrinsic::s390_vuplhw:
5512 case Intrinsic::s390_vuplf:
5513 case Intrinsic::s390_vuplg:
5514 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5515 Operand: Op.getOperand(i: 1));
5516
5517 case Intrinsic::s390_vupllb:
5518 case Intrinsic::s390_vupllh:
5519 case Intrinsic::s390_vupllf:
5520 case Intrinsic::s390_vupllg:
5521 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5522 Operand: Op.getOperand(i: 1));
5523
5524 case Intrinsic::s390_vsumb:
5525 case Intrinsic::s390_vsumh:
5526 case Intrinsic::s390_vsumgh:
5527 case Intrinsic::s390_vsumgf:
5528 case Intrinsic::s390_vsumqf:
5529 case Intrinsic::s390_vsumqg:
5530 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5531 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5532
5533 case Intrinsic::s390_vaq:
5534 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5535 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5536 case Intrinsic::s390_vaccb:
5537 case Intrinsic::s390_vacch:
5538 case Intrinsic::s390_vaccf:
5539 case Intrinsic::s390_vaccg:
5540 case Intrinsic::s390_vaccq:
5541 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5542 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5543 case Intrinsic::s390_vacq:
5544 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5545 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5546 case Intrinsic::s390_vacccq:
5547 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5548 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5549
5550 case Intrinsic::s390_vsq:
5551 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5552 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5553 case Intrinsic::s390_vscbib:
5554 case Intrinsic::s390_vscbih:
5555 case Intrinsic::s390_vscbif:
5556 case Intrinsic::s390_vscbig:
5557 case Intrinsic::s390_vscbiq:
5558 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5559 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5560 case Intrinsic::s390_vsbiq:
5561 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5562 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5563 case Intrinsic::s390_vsbcbiq:
5564 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5565 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5566
5567 case Intrinsic::s390_vmhb:
5568 case Intrinsic::s390_vmhh:
5569 case Intrinsic::s390_vmhf:
5570 case Intrinsic::s390_vmhg:
5571 case Intrinsic::s390_vmhq:
5572 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5573 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5574 case Intrinsic::s390_vmlhb:
5575 case Intrinsic::s390_vmlhh:
5576 case Intrinsic::s390_vmlhf:
5577 case Intrinsic::s390_vmlhg:
5578 case Intrinsic::s390_vmlhq:
5579 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5580 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5581
5582 case Intrinsic::s390_vmahb:
5583 case Intrinsic::s390_vmahh:
5584 case Intrinsic::s390_vmahf:
5585 case Intrinsic::s390_vmahg:
5586 case Intrinsic::s390_vmahq:
5587 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5588 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5589 case Intrinsic::s390_vmalhb:
5590 case Intrinsic::s390_vmalhh:
5591 case Intrinsic::s390_vmalhf:
5592 case Intrinsic::s390_vmalhg:
5593 case Intrinsic::s390_vmalhq:
5594 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5595 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5596
5597 case Intrinsic::s390_vmeb:
5598 case Intrinsic::s390_vmeh:
5599 case Intrinsic::s390_vmef:
5600 case Intrinsic::s390_vmeg:
5601 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5602 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5603 case Intrinsic::s390_vmleb:
5604 case Intrinsic::s390_vmleh:
5605 case Intrinsic::s390_vmlef:
5606 case Intrinsic::s390_vmleg:
5607 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5608 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5609 case Intrinsic::s390_vmob:
5610 case Intrinsic::s390_vmoh:
5611 case Intrinsic::s390_vmof:
5612 case Intrinsic::s390_vmog:
5613 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5614 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5615 case Intrinsic::s390_vmlob:
5616 case Intrinsic::s390_vmloh:
5617 case Intrinsic::s390_vmlof:
5618 case Intrinsic::s390_vmlog:
5619 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5620 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5621
5622 case Intrinsic::s390_vmaeb:
5623 case Intrinsic::s390_vmaeh:
5624 case Intrinsic::s390_vmaef:
5625 case Intrinsic::s390_vmaeg:
5626 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5627 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5628 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5629 N2: Op.getOperand(i: 3));
5630 case Intrinsic::s390_vmaleb:
5631 case Intrinsic::s390_vmaleh:
5632 case Intrinsic::s390_vmalef:
5633 case Intrinsic::s390_vmaleg:
5634 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5635 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5636 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5637 N2: Op.getOperand(i: 3));
5638 case Intrinsic::s390_vmaob:
5639 case Intrinsic::s390_vmaoh:
5640 case Intrinsic::s390_vmaof:
5641 case Intrinsic::s390_vmaog:
5642 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5643 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5644 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5645 N2: Op.getOperand(i: 3));
5646 case Intrinsic::s390_vmalob:
5647 case Intrinsic::s390_vmaloh:
5648 case Intrinsic::s390_vmalof:
5649 case Intrinsic::s390_vmalog:
5650 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5651 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5652 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5653 N2: Op.getOperand(i: 3));
5654 }
5655
5656 return SDValue();
5657}
5658
5659namespace {
5660// Says that SystemZISD operation Opcode can be used to perform the equivalent
5661// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5662// Operand is the constant third operand, otherwise it is the number of
5663// bytes in each element of the result.
5664struct Permute {
5665 unsigned Opcode;
5666 unsigned Operand;
5667 unsigned char Bytes[SystemZ::VectorBytes];
5668};
5669}
5670
5671static const Permute PermuteForms[] = {
5672 // VMRHG
5673 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5674 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5675 // VMRHF
5676 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5677 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5678 // VMRHH
5679 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5680 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5681 // VMRHB
5682 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5683 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5684 // VMRLG
5685 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5686 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5687 // VMRLF
5688 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5689 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5690 // VMRLH
5691 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5692 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5693 // VMRLB
5694 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5695 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5696 // VPKG
5697 { .Opcode: SystemZISD::PACK, .Operand: 4,
5698 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5699 // VPKF
5700 { .Opcode: SystemZISD::PACK, .Operand: 2,
5701 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5702 // VPKH
5703 { .Opcode: SystemZISD::PACK, .Operand: 1,
5704 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5705 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5706 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5707 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5708 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5709 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5710 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5711};
5712
5713// Called after matching a vector shuffle against a particular pattern.
5714// Both the original shuffle and the pattern have two vector operands.
5715// OpNos[0] is the operand of the original shuffle that should be used for
5716// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5717// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5718// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5719// for operands 0 and 1 of the pattern.
5720static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5721 if (OpNos[0] < 0) {
5722 if (OpNos[1] < 0)
5723 return false;
5724 OpNo0 = OpNo1 = OpNos[1];
5725 } else if (OpNos[1] < 0) {
5726 OpNo0 = OpNo1 = OpNos[0];
5727 } else {
5728 OpNo0 = OpNos[0];
5729 OpNo1 = OpNos[1];
5730 }
5731 return true;
5732}
5733
5734// Bytes is a VPERM-like permute vector, except that -1 is used for
5735// undefined bytes. Return true if the VPERM can be implemented using P.
5736// When returning true set OpNo0 to the VPERM operand that should be
5737// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5738//
5739// For example, if swapping the VPERM operands allows P to match, OpNo0
5740// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5741// operand, but rewriting it to use two duplicated operands allows it to
5742// match P, then OpNo0 and OpNo1 will be the same.
5743static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5744 unsigned &OpNo0, unsigned &OpNo1) {
5745 int OpNos[] = { -1, -1 };
5746 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5747 int Elt = Bytes[I];
5748 if (Elt >= 0) {
5749 // Make sure that the two permute vectors use the same suboperand
5750 // byte number. Only the operand numbers (the high bits) are
5751 // allowed to differ.
5752 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5753 return false;
5754 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5755 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5756 // Make sure that the operand mappings are consistent with previous
5757 // elements.
5758 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5759 return false;
5760 OpNos[ModelOpNo] = RealOpNo;
5761 }
5762 }
5763 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5764}
5765
5766// As above, but search for a matching permute.
5767static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5768 unsigned &OpNo0, unsigned &OpNo1) {
5769 for (auto &P : PermuteForms)
5770 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5771 return &P;
5772 return nullptr;
5773}
5774
5775// Bytes is a VPERM-like permute vector, except that -1 is used for
5776// undefined bytes. This permute is an operand of an outer permute.
5777// See whether redistributing the -1 bytes gives a shuffle that can be
5778// implemented using P. If so, set Transform to a VPERM-like permute vector
5779// that, when applied to the result of P, gives the original permute in Bytes.
5780static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5781 const Permute &P,
5782 SmallVectorImpl<int> &Transform) {
5783 unsigned To = 0;
5784 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5785 int Elt = Bytes[From];
5786 if (Elt < 0)
5787 // Byte number From of the result is undefined.
5788 Transform[From] = -1;
5789 else {
5790 while (P.Bytes[To] != Elt) {
5791 To += 1;
5792 if (To == SystemZ::VectorBytes)
5793 return false;
5794 }
5795 Transform[From] = To;
5796 }
5797 }
5798 return true;
5799}
5800
5801// As above, but search for a matching permute.
5802static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5803 SmallVectorImpl<int> &Transform) {
5804 for (auto &P : PermuteForms)
5805 if (matchDoublePermute(Bytes, P, Transform))
5806 return &P;
5807 return nullptr;
5808}
5809
5810// Convert the mask of the given shuffle op into a byte-level mask,
5811// as if it had type vNi8.
5812static bool getVPermMask(SDValue ShuffleOp,
5813 SmallVectorImpl<int> &Bytes) {
5814 EVT VT = ShuffleOp.getValueType();
5815 unsigned NumElements = VT.getVectorNumElements();
5816 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5817
5818 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5819 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5820 for (unsigned I = 0; I < NumElements; ++I) {
5821 int Index = VSN->getMaskElt(Idx: I);
5822 if (Index >= 0)
5823 for (unsigned J = 0; J < BytesPerElement; ++J)
5824 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5825 }
5826 return true;
5827 }
5828 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5829 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5830 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5831 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5832 for (unsigned I = 0; I < NumElements; ++I)
5833 for (unsigned J = 0; J < BytesPerElement; ++J)
5834 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5835 return true;
5836 }
5837 return false;
5838}
5839
5840// Bytes is a VPERM-like permute vector, except that -1 is used for
5841// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5842// the result come from a contiguous sequence of bytes from one input.
5843// Set Base to the selector for the first byte if so.
5844static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5845 unsigned BytesPerElement, int &Base) {
5846 Base = -1;
5847 for (unsigned I = 0; I < BytesPerElement; ++I) {
5848 if (Bytes[Start + I] >= 0) {
5849 unsigned Elem = Bytes[Start + I];
5850 if (Base < 0) {
5851 Base = Elem - I;
5852 // Make sure the bytes would come from one input operand.
5853 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5854 return false;
5855 } else if (unsigned(Base) != Elem - I)
5856 return false;
5857 }
5858 }
5859 return true;
5860}
5861
5862// Bytes is a VPERM-like permute vector, except that -1 is used for
5863// undefined bytes. Return true if it can be performed using VSLDB.
5864// When returning true, set StartIndex to the shift amount and OpNo0
5865// and OpNo1 to the VPERM operands that should be used as the first
5866// and second shift operand respectively.
5867static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5868 unsigned &StartIndex, unsigned &OpNo0,
5869 unsigned &OpNo1) {
5870 int OpNos[] = { -1, -1 };
5871 int Shift = -1;
5872 for (unsigned I = 0; I < 16; ++I) {
5873 int Index = Bytes[I];
5874 if (Index >= 0) {
5875 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5876 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5877 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5878 if (Shift < 0)
5879 Shift = ExpectedShift;
5880 else if (Shift != ExpectedShift)
5881 return false;
5882 // Make sure that the operand mappings are consistent with previous
5883 // elements.
5884 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5885 return false;
5886 OpNos[ModelOpNo] = RealOpNo;
5887 }
5888 }
5889 StartIndex = Shift;
5890 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5891}
5892
5893// Create a node that performs P on operands Op0 and Op1, casting the
5894// operands to the appropriate type. The type of the result is determined by P.
5895static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5896 const Permute &P, SDValue Op0, SDValue Op1) {
5897 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5898 // elements of a PACK are twice as wide as the outputs.
5899 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5900 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5901 P.Operand);
5902 // Cast both operands to the appropriate type.
5903 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
5904 NumElements: SystemZ::VectorBytes / InBytes);
5905 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
5906 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
5907 SDValue Op;
5908 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5909 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
5910 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
5911 } else if (P.Opcode == SystemZISD::PACK) {
5912 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
5913 NumElements: SystemZ::VectorBytes / P.Operand);
5914 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
5915 } else {
5916 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
5917 }
5918 return Op;
5919}
5920
5921static bool isZeroVector(SDValue N) {
5922 if (N->getOpcode() == ISD::BITCAST)
5923 N = N->getOperand(Num: 0);
5924 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5925 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
5926 return Op->getZExtValue() == 0;
5927 return ISD::isBuildVectorAllZeros(N: N.getNode());
5928}
5929
5930// Return the index of the zero/undef vector, or UINT32_MAX if not found.
5931static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5932 for (unsigned I = 0; I < Num ; I++)
5933 if (isZeroVector(N: Ops[I]))
5934 return I;
5935 return UINT32_MAX;
5936}
5937
5938// Bytes is a VPERM-like permute vector, except that -1 is used for
5939// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5940// VSLDB or VPERM.
5941static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5942 SDValue *Ops,
5943 const SmallVectorImpl<int> &Bytes) {
5944 for (unsigned I = 0; I < 2; ++I)
5945 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
5946
5947 // First see whether VSLDB can be used.
5948 unsigned StartIndex, OpNo0, OpNo1;
5949 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5950 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
5951 N2: Ops[OpNo1],
5952 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
5953
5954 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5955 // eliminate a zero vector by reusing any zero index in the permute vector.
5956 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
5957 if (ZeroVecIdx != UINT32_MAX) {
5958 bool MaskFirst = true;
5959 int ZeroIdx = -1;
5960 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5961 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5962 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5963 if (OpNo == ZeroVecIdx && I == 0) {
5964 // If the first byte is zero, use mask as first operand.
5965 ZeroIdx = 0;
5966 break;
5967 }
5968 if (OpNo != ZeroVecIdx && Byte == 0) {
5969 // If mask contains a zero, use it by placing that vector first.
5970 ZeroIdx = I + SystemZ::VectorBytes;
5971 MaskFirst = false;
5972 break;
5973 }
5974 }
5975 if (ZeroIdx != -1) {
5976 SDValue IndexNodes[SystemZ::VectorBytes];
5977 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5978 if (Bytes[I] >= 0) {
5979 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5980 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5981 if (OpNo == ZeroVecIdx)
5982 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
5983 else {
5984 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
5985 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
5986 }
5987 } else
5988 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5989 }
5990 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5991 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5992 if (MaskFirst)
5993 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
5994 N3: Mask);
5995 else
5996 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
5997 N3: Mask);
5998 }
5999 }
6000
6001 SDValue IndexNodes[SystemZ::VectorBytes];
6002 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6003 if (Bytes[I] >= 0)
6004 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
6005 else
6006 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
6007 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
6008 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
6009 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
6010}
6011
6012namespace {
6013// Describes a general N-operand vector shuffle.
6014struct GeneralShuffle {
6015 GeneralShuffle(EVT vt)
6016 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
6017 void addUndef();
6018 bool add(SDValue, unsigned);
6019 SDValue getNode(SelectionDAG &, const SDLoc &);
6020 void tryPrepareForUnpack();
6021 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
6022 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
6023
6024 // The operands of the shuffle.
6025 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
6026
6027 // Index I is -1 if byte I of the result is undefined. Otherwise the
6028 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
6029 // Bytes[I] / SystemZ::VectorBytes.
6030 SmallVector<int, SystemZ::VectorBytes> Bytes;
6031
6032 // The type of the shuffle result.
6033 EVT VT;
6034
6035 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
6036 unsigned UnpackFromEltSize;
6037 // True if the final unpack uses the low half.
6038 bool UnpackLow;
6039};
6040} // namespace
6041
6042// Add an extra undefined element to the shuffle.
6043void GeneralShuffle::addUndef() {
6044 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6045 for (unsigned I = 0; I < BytesPerElement; ++I)
6046 Bytes.push_back(Elt: -1);
6047}
6048
6049// Add an extra element to the shuffle, taking it from element Elem of Op.
6050// A null Op indicates a vector input whose value will be calculated later;
6051// there is at most one such input per shuffle and it always has the same
6052// type as the result. Aborts and returns false if the source vector elements
6053// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
6054// LLVM they become implicitly extended, but this is rare and not optimized.
6055bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
6056 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6057
6058 // The source vector can have wider elements than the result,
6059 // either through an explicit TRUNCATE or because of type legalization.
6060 // We want the least significant part.
6061 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
6062 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
6063
6064 // Return false if the source elements are smaller than their destination
6065 // elements.
6066 if (FromBytesPerElement < BytesPerElement)
6067 return false;
6068
6069 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6070 (FromBytesPerElement - BytesPerElement));
6071
6072 // Look through things like shuffles and bitcasts.
6073 while (Op.getNode()) {
6074 if (Op.getOpcode() == ISD::BITCAST)
6075 Op = Op.getOperand(i: 0);
6076 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6077 // See whether the bytes we need come from a contiguous part of one
6078 // operand.
6079 SmallVector<int, SystemZ::VectorBytes> OpBytes;
6080 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
6081 break;
6082 int NewByte;
6083 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
6084 break;
6085 if (NewByte < 0) {
6086 addUndef();
6087 return true;
6088 }
6089 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
6090 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6091 } else if (Op.isUndef()) {
6092 addUndef();
6093 return true;
6094 } else
6095 break;
6096 }
6097
6098 // Make sure that the source of the extraction is in Ops.
6099 unsigned OpNo = 0;
6100 for (; OpNo < Ops.size(); ++OpNo)
6101 if (Ops[OpNo] == Op)
6102 break;
6103 if (OpNo == Ops.size())
6104 Ops.push_back(Elt: Op);
6105
6106 // Add the element to Bytes.
6107 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6108 for (unsigned I = 0; I < BytesPerElement; ++I)
6109 Bytes.push_back(Elt: Base + I);
6110
6111 return true;
6112}
6113
6114// Return SDNodes for the completed shuffle.
6115SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6116 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6117
6118 if (Ops.size() == 0)
6119 return DAG.getUNDEF(VT);
6120
6121 // Use a single unpack if possible as the last operation.
6122 tryPrepareForUnpack();
6123
6124 // Make sure that there are at least two shuffle operands.
6125 if (Ops.size() == 1)
6126 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
6127
6128 // Create a tree of shuffles, deferring root node until after the loop.
6129 // Try to redistribute the undefined elements of non-root nodes so that
6130 // the non-root shuffles match something like a pack or merge, then adjust
6131 // the parent node's permute vector to compensate for the new order.
6132 // Among other things, this copes with vectors like <2 x i16> that were
6133 // padded with undefined elements during type legalization.
6134 //
6135 // In the best case this redistribution will lead to the whole tree
6136 // using packs and merges. It should rarely be a loss in other cases.
6137 unsigned Stride = 1;
6138 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6139 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6140 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6141
6142 // Create a mask for just these two operands.
6143 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
6144 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6145 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6146 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6147 if (OpNo == I)
6148 NewBytes[J] = Byte;
6149 else if (OpNo == I + Stride)
6150 NewBytes[J] = SystemZ::VectorBytes + Byte;
6151 else
6152 NewBytes[J] = -1;
6153 }
6154 // See if it would be better to reorganize NewMask to avoid using VPERM.
6155 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6156 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6157 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6158 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6159 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6160 if (NewBytes[J] >= 0) {
6161 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6162 "Invalid double permute");
6163 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6164 } else
6165 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6166 }
6167 } else {
6168 // Just use NewBytes on the operands.
6169 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6170 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6171 if (NewBytes[J] >= 0)
6172 Bytes[J] = I * SystemZ::VectorBytes + J;
6173 }
6174 }
6175 }
6176
6177 // Now we just have 2 inputs. Put the second operand in Ops[1].
6178 if (Stride > 1) {
6179 Ops[1] = Ops[Stride];
6180 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6181 if (Bytes[I] >= int(SystemZ::VectorBytes))
6182 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6183 }
6184
6185 // Look for an instruction that can do the permute without resorting
6186 // to VPERM.
6187 unsigned OpNo0, OpNo1;
6188 SDValue Op;
6189 if (unpackWasPrepared() && Ops[1].isUndef())
6190 Op = Ops[0];
6191 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6192 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6193 else
6194 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6195
6196 Op = insertUnpackIfPrepared(DAG, DL, Op);
6197
6198 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6199}
6200
6201#ifndef NDEBUG
6202static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6203 dbgs() << Msg.c_str() << " { ";
6204 for (unsigned I = 0; I < Bytes.size(); I++)
6205 dbgs() << Bytes[I] << " ";
6206 dbgs() << "}\n";
6207}
6208#endif
6209
6210// If the Bytes vector matches an unpack operation, prepare to do the unpack
6211// after all else by removing the zero vector and the effect of the unpack on
6212// Bytes.
6213void GeneralShuffle::tryPrepareForUnpack() {
6214 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6215 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6216 return;
6217
6218 // Only do this if removing the zero vector reduces the depth, otherwise
6219 // the critical path will increase with the final unpack.
6220 if (Ops.size() > 2 &&
6221 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6222 return;
6223
6224 // Find an unpack that would allow removing the zero vector from Ops.
6225 UnpackFromEltSize = 1;
6226 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6227 bool MatchUnpack = true;
6228 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6229 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6230 unsigned ToEltSize = UnpackFromEltSize * 2;
6231 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6232 if (!IsZextByte)
6233 SrcBytes.push_back(Elt: Bytes[Elt]);
6234 if (Bytes[Elt] != -1) {
6235 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6236 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6237 MatchUnpack = false;
6238 break;
6239 }
6240 }
6241 }
6242 if (MatchUnpack) {
6243 if (Ops.size() == 2) {
6244 // Don't use unpack if a single source operand needs rearrangement.
6245 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6246 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6247 if (SrcBytes[i] == -1)
6248 continue;
6249 if (SrcBytes[i] % 16 != int(i))
6250 CanUseUnpackHigh = false;
6251 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6252 CanUseUnpackLow = false;
6253 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6254 UnpackFromEltSize = UINT_MAX;
6255 return;
6256 }
6257 }
6258 if (!CanUseUnpackHigh)
6259 UnpackLow = true;
6260 }
6261 break;
6262 }
6263 }
6264 if (UnpackFromEltSize > 4)
6265 return;
6266
6267 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6268 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6269 << ".\n";
6270 dumpBytes(Bytes, "Original Bytes vector:"););
6271
6272 // Apply the unpack in reverse to the Bytes array.
6273 unsigned B = 0;
6274 if (UnpackLow) {
6275 while (B < SystemZ::VectorBytes / 2)
6276 Bytes[B++] = -1;
6277 }
6278 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6279 Elt += UnpackFromEltSize;
6280 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6281 Bytes[B] = Bytes[Elt];
6282 }
6283 if (!UnpackLow) {
6284 while (B < SystemZ::VectorBytes)
6285 Bytes[B++] = -1;
6286 }
6287
6288 // Remove the zero vector from Ops
6289 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6290 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6291 if (Bytes[I] >= 0) {
6292 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6293 if (OpNo > ZeroVecOpNo)
6294 Bytes[I] -= SystemZ::VectorBytes;
6295 }
6296
6297 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6298 dbgs() << "\n";);
6299}
6300
6301SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6302 const SDLoc &DL,
6303 SDValue Op) {
6304 if (!unpackWasPrepared())
6305 return Op;
6306 unsigned InBits = UnpackFromEltSize * 8;
6307 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6308 NumElements: SystemZ::VectorBits / InBits);
6309 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6310 unsigned OutBits = InBits * 2;
6311 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6312 NumElements: SystemZ::VectorBits / OutBits);
6313 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6314 : SystemZISD::UNPACKL_HIGH,
6315 DL, VT: OutVT, Operand: PackedOp);
6316}
6317
6318// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6319static bool isScalarToVector(SDValue Op) {
6320 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6321 if (!Op.getOperand(i: I).isUndef())
6322 return false;
6323 return true;
6324}
6325
6326// Return a vector of type VT that contains Value in the first element.
6327// The other elements don't matter.
6328static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6329 SDValue Value) {
6330 // If we have a constant, replicate it to all elements and let the
6331 // BUILD_VECTOR lowering take care of it.
6332 if (Value.getOpcode() == ISD::Constant ||
6333 Value.getOpcode() == ISD::ConstantFP) {
6334 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6335 return DAG.getBuildVector(VT, DL, Ops);
6336 }
6337 if (Value.isUndef())
6338 return DAG.getUNDEF(VT);
6339 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6340}
6341
6342// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6343// element 1. Used for cases in which replication is cheap.
6344static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6345 SDValue Op0, SDValue Op1) {
6346 if (Op0.isUndef()) {
6347 if (Op1.isUndef())
6348 return DAG.getUNDEF(VT);
6349 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6350 }
6351 if (Op1.isUndef())
6352 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6353 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6354 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6355 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6356}
6357
6358// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6359// vector for them.
6360static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6361 SDValue Op1) {
6362 if (Op0.isUndef() && Op1.isUndef())
6363 return DAG.getUNDEF(VT: MVT::v2i64);
6364 // If one of the two inputs is undefined then replicate the other one,
6365 // in order to avoid using another register unnecessarily.
6366 if (Op0.isUndef())
6367 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6368 else if (Op1.isUndef())
6369 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6370 else {
6371 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6372 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6373 }
6374 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6375}
6376
6377// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6378// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6379// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6380// would benefit from this representation and return it if so.
6381static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6382 BuildVectorSDNode *BVN) {
6383 EVT VT = BVN->getValueType(ResNo: 0);
6384 unsigned NumElements = VT.getVectorNumElements();
6385
6386 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6387 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6388 // need a BUILD_VECTOR, add an additional placeholder operand for that
6389 // BUILD_VECTOR and store its operands in ResidueOps.
6390 GeneralShuffle GS(VT);
6391 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6392 bool FoundOne = false;
6393 for (unsigned I = 0; I < NumElements; ++I) {
6394 SDValue Op = BVN->getOperand(Num: I);
6395 if (Op.getOpcode() == ISD::TRUNCATE)
6396 Op = Op.getOperand(i: 0);
6397 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6398 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6399 unsigned Elem = Op.getConstantOperandVal(i: 1);
6400 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6401 return SDValue();
6402 FoundOne = true;
6403 } else if (Op.isUndef()) {
6404 GS.addUndef();
6405 } else {
6406 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6407 return SDValue();
6408 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6409 }
6410 }
6411
6412 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6413 if (!FoundOne)
6414 return SDValue();
6415
6416 // Create the BUILD_VECTOR for the remaining elements, if any.
6417 if (!ResidueOps.empty()) {
6418 while (ResidueOps.size() < NumElements)
6419 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6420 for (auto &Op : GS.Ops) {
6421 if (!Op.getNode()) {
6422 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6423 break;
6424 }
6425 }
6426 }
6427 return GS.getNode(DAG, DL: SDLoc(BVN));
6428}
6429
6430bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6431 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6432 return true;
6433 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6434 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6435 return true;
6436 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6437 return true;
6438 return false;
6439}
6440
6441static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL,
6442 unsigned MergedBits, EVT VT, SDValue Op0,
6443 SDValue Op1) {
6444 MVT IntVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MergedBits),
6445 NumElements: SystemZ::VectorBits / MergedBits);
6446 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6447 "Handling full vectors only.");
6448 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0);
6449 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op1);
6450 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT: IntVecVT, N1: Op0, N2: Op1);
6451 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6452}
6453
6454static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL,
6455 EVT VT, SmallVectorImpl<SDValue> &Elems,
6456 unsigned Pos) {
6457 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 0], Op1: Elems[Pos + 1]);
6458 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 2], Op1: Elems[Pos + 3]);
6459 // Avoid unnecessary undefs by reusing the other operand.
6460 if (Op01.isUndef()) {
6461 if (Op23.isUndef())
6462 return Op01;
6463 Op01 = Op23;
6464 } else if (Op23.isUndef())
6465 Op23 = Op01;
6466 // Merging identical replications is a no-op.
6467 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6468 return Op01;
6469 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6470 return mergeHighParts(DAG, DL, MergedBits, VT, Op0: Op01, Op1: Op23);
6471}
6472
6473// Combine GPR scalar values Elems into a vector of type VT.
6474SDValue
6475SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6476 SmallVectorImpl<SDValue> &Elems) const {
6477 // See whether there is a single replicated value.
6478 SDValue Single;
6479 unsigned int NumElements = Elems.size();
6480 unsigned int Count = 0;
6481 for (auto Elem : Elems) {
6482 if (!Elem.isUndef()) {
6483 if (!Single.getNode())
6484 Single = Elem;
6485 else if (Elem != Single) {
6486 Single = SDValue();
6487 break;
6488 }
6489 Count += 1;
6490 }
6491 }
6492 // There are three cases here:
6493 //
6494 // - if the only defined element is a loaded one, the best sequence
6495 // is a replicating load.
6496 //
6497 // - otherwise, if the only defined element is an i64 value, we will
6498 // end up with the same VLVGP sequence regardless of whether we short-cut
6499 // for replication or fall through to the later code.
6500 //
6501 // - otherwise, if the only defined element is an i32 or smaller value,
6502 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6503 // This is only a win if the single defined element is used more than once.
6504 // In other cases we're better off using a single VLVGx.
6505 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6506 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6507
6508 // If all elements are loads, use VLREP/VLEs (below).
6509 bool AllLoads = true;
6510 for (auto Elem : Elems)
6511 if (!isVectorElementLoad(Op: Elem)) {
6512 AllLoads = false;
6513 break;
6514 }
6515
6516 // The best way of building a v2i64 from two i64s is to use VLVGP.
6517 if (VT == MVT::v2i64 && !AllLoads)
6518 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6519
6520 // Use a 64-bit merge high to combine two doubles.
6521 if (VT == MVT::v2f64 && !AllLoads)
6522 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6523
6524 // Build v4f32 values directly from the FPRs:
6525 //
6526 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6527 // V V VMRHF
6528 // <ABxx> <CDxx>
6529 // V VMRHG
6530 // <ABCD>
6531 if (VT == MVT::v4f32 && !AllLoads)
6532 return buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6533
6534 // Same for v8f16.
6535 if (VT == MVT::v8f16 && !AllLoads) {
6536 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6537 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 4);
6538 // Avoid unnecessary undefs by reusing the other operand.
6539 if (Op0123.isUndef())
6540 Op0123 = Op4567;
6541 else if (Op4567.isUndef())
6542 Op4567 = Op0123;
6543 // Merging identical replications is a no-op.
6544 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6545 return Op0123;
6546 return mergeHighParts(DAG, DL, MergedBits: 64, VT, Op0: Op0123, Op1: Op4567);
6547 }
6548
6549 // Collect the constant terms.
6550 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6551 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6552
6553 unsigned NumConstants = 0;
6554 for (unsigned I = 0; I < NumElements; ++I) {
6555 SDValue Elem = Elems[I];
6556 if (Elem.getOpcode() == ISD::Constant ||
6557 Elem.getOpcode() == ISD::ConstantFP) {
6558 NumConstants += 1;
6559 Constants[I] = Elem;
6560 Done[I] = true;
6561 }
6562 }
6563 // If there was at least one constant, fill in the other elements of
6564 // Constants with undefs to get a full vector constant and use that
6565 // as the starting point.
6566 SDValue Result;
6567 SDValue ReplicatedVal;
6568 if (NumConstants > 0) {
6569 for (unsigned I = 0; I < NumElements; ++I)
6570 if (!Constants[I].getNode())
6571 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6572 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6573 } else {
6574 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6575 // avoid a false dependency on any previous contents of the vector
6576 // register.
6577
6578 // Use a VLREP if at least one element is a load. Make sure to replicate
6579 // the load with the most elements having its value.
6580 std::map<const SDNode*, unsigned> UseCounts;
6581 SDNode *LoadMaxUses = nullptr;
6582 for (unsigned I = 0; I < NumElements; ++I)
6583 if (isVectorElementLoad(Op: Elems[I])) {
6584 SDNode *Ld = Elems[I].getNode();
6585 unsigned Count = ++UseCounts[Ld];
6586 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6587 LoadMaxUses = Ld;
6588 }
6589 if (LoadMaxUses != nullptr) {
6590 ReplicatedVal = SDValue(LoadMaxUses, 0);
6591 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6592 } else {
6593 // Try to use VLVGP.
6594 unsigned I1 = NumElements / 2 - 1;
6595 unsigned I2 = NumElements - 1;
6596 bool Def1 = !Elems[I1].isUndef();
6597 bool Def2 = !Elems[I2].isUndef();
6598 if (Def1 || Def2) {
6599 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6600 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6601 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6602 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6603 Done[I1] = true;
6604 Done[I2] = true;
6605 } else
6606 Result = DAG.getUNDEF(VT);
6607 }
6608 }
6609
6610 // Use VLVGx to insert the other elements.
6611 for (unsigned I = 0; I < NumElements; ++I)
6612 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6613 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6614 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6615 return Result;
6616}
6617
6618SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6619 SelectionDAG &DAG) const {
6620 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6621 SDLoc DL(Op);
6622 EVT VT = Op.getValueType();
6623
6624 if (BVN->isConstant()) {
6625 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6626 return Op;
6627
6628 // Fall back to loading it from memory.
6629 return SDValue();
6630 }
6631
6632 // See if we should use shuffles to construct the vector from other vectors.
6633 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6634 return Res;
6635
6636 // Detect SCALAR_TO_VECTOR conversions.
6637 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6638 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6639
6640 // Otherwise use buildVector to build the vector up from GPRs.
6641 unsigned NumElements = Op.getNumOperands();
6642 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6643 for (unsigned I = 0; I < NumElements; ++I)
6644 Ops[I] = Op.getOperand(i: I);
6645 return buildVector(DAG, DL, VT, Elems&: Ops);
6646}
6647
6648SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6649 SelectionDAG &DAG) const {
6650 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6651 SDLoc DL(Op);
6652 EVT VT = Op.getValueType();
6653 unsigned NumElements = VT.getVectorNumElements();
6654
6655 if (VSN->isSplat()) {
6656 SDValue Op0 = Op.getOperand(i: 0);
6657 unsigned Index = VSN->getSplatIndex();
6658 assert(Index < VT.getVectorNumElements() &&
6659 "Splat index should be defined and in first operand");
6660 // See whether the value we're splatting is directly available as a scalar.
6661 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6662 Op0.getOpcode() == ISD::BUILD_VECTOR)
6663 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6664 // Otherwise keep it as a vector-to-vector operation.
6665 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6666 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6667 }
6668
6669 GeneralShuffle GS(VT);
6670 for (unsigned I = 0; I < NumElements; ++I) {
6671 int Elt = VSN->getMaskElt(Idx: I);
6672 if (Elt < 0)
6673 GS.addUndef();
6674 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6675 Elem: unsigned(Elt) % NumElements))
6676 return SDValue();
6677 }
6678 return GS.getNode(DAG, DL: SDLoc(VSN));
6679}
6680
6681SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6682 SelectionDAG &DAG) const {
6683 SDLoc DL(Op);
6684 // Just insert the scalar into element 0 of an undefined vector.
6685 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6686 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6687 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6688}
6689
6690// Shift the lower 2 bytes of Op to the left in order to insert into the
6691// upper 2 bytes of the FP register.
6692static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6693 assert(Op.getSimpleValueType() == MVT::i64 &&
6694 "Expexted to convert i64 to f16.");
6695 SDLoc DL(Op);
6696 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6697 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6698 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6699 SDValue F16Val =
6700 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6701 return F16Val;
6702}
6703
6704// Extract Op into GPR and shift the 2 f16 bytes to the right.
6705static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6706 assert(Op.getSimpleValueType() == MVT::f16 &&
6707 "Expected to convert f16 to i64.");
6708 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6709 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6710 Operand: SDValue(U32, 0), Subreg: Op);
6711 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6712 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6713 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6714 return Shft;
6715}
6716
6717SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6718 SelectionDAG &DAG) const {
6719 // Handle insertions of floating-point values.
6720 SDLoc DL(Op);
6721 SDValue Op0 = Op.getOperand(i: 0);
6722 SDValue Op1 = Op.getOperand(i: 1);
6723 SDValue Op2 = Op.getOperand(i: 2);
6724 EVT VT = Op.getValueType();
6725
6726 // Insertions into constant indices of a v2f64 can be done using VPDI.
6727 // However, if the inserted value is a bitcast or a constant then it's
6728 // better to use GPRs, as below.
6729 if (VT == MVT::v2f64 &&
6730 Op1.getOpcode() != ISD::BITCAST &&
6731 Op1.getOpcode() != ISD::ConstantFP &&
6732 Op2.getOpcode() == ISD::Constant) {
6733 uint64_t Index = Op2->getAsZExtVal();
6734 unsigned Mask = VT.getVectorNumElements() - 1;
6735 if (Index <= Mask)
6736 return Op;
6737 }
6738
6739 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6740 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6741 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6742 SDValue IntOp1 =
6743 VT == MVT::v8f16
6744 ? DAG.getZExtOrTrunc(Op: convertFromF16(Op: Op1, DL, DAG), DL, VT: MVT::i32)
6745 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1);
6746 SDValue Res =
6747 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6748 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: IntOp1, N3: Op2);
6749 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6750}
6751
6752SDValue
6753SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6754 SelectionDAG &DAG) const {
6755 // Handle extractions of floating-point values.
6756 SDLoc DL(Op);
6757 SDValue Op0 = Op.getOperand(i: 0);
6758 SDValue Op1 = Op.getOperand(i: 1);
6759 EVT VT = Op.getValueType();
6760 EVT VecVT = Op0.getValueType();
6761
6762 // Extractions of constant indices can be done directly.
6763 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Val&: Op1)) {
6764 uint64_t Index = CIndexN->getZExtValue();
6765 unsigned Mask = VecVT.getVectorNumElements() - 1;
6766 if (Index <= Mask)
6767 return Op;
6768 }
6769
6770 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6771 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6772 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6773 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6774 SDValue Extr = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ExtrVT,
6775 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6776 if (VT == MVT::f16)
6777 return convertToF16(Op: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Extr), DAG);
6778 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Extr);
6779}
6780
6781SDValue SystemZTargetLowering::
6782lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6783 SDValue PackedOp = Op.getOperand(i: 0);
6784 EVT OutVT = Op.getValueType();
6785 EVT InVT = PackedOp.getValueType();
6786 unsigned ToBits = OutVT.getScalarSizeInBits();
6787 unsigned FromBits = InVT.getScalarSizeInBits();
6788 unsigned StartOffset = 0;
6789
6790 // If the input is a VECTOR_SHUFFLE, there are a number of important
6791 // cases where we can directly implement the sign-extension of the
6792 // original input lanes of the shuffle.
6793 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6794 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6795 ArrayRef<int> ShuffleMask = SVN->getMask();
6796 int OutNumElts = OutVT.getVectorNumElements();
6797
6798 // Recognize the special case where the sign-extension can be done
6799 // by the VSEG instruction. Handled via the default expander.
6800 if (ToBits == 64 && OutNumElts == 2) {
6801 int NumElem = ToBits / FromBits;
6802 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6803 return SDValue();
6804 }
6805
6806 // Recognize the special case where we can fold the shuffle by
6807 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6808 int StartOffsetCandidate = -1;
6809 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6810 if (ShuffleMask[Elt] == -1)
6811 continue;
6812 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6813 if (StartOffsetCandidate == -1)
6814 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6815 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6816 continue;
6817 }
6818 StartOffsetCandidate = -1;
6819 break;
6820 }
6821 if (StartOffsetCandidate != -1) {
6822 StartOffset = StartOffsetCandidate;
6823 PackedOp = PackedOp.getOperand(i: 0);
6824 }
6825 }
6826
6827 do {
6828 FromBits *= 2;
6829 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6830 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6831 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6832 if (StartOffset >= OutNumElts) {
6833 Opcode = SystemZISD::UNPACK_LOW;
6834 StartOffset -= OutNumElts;
6835 }
6836 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6837 } while (FromBits != ToBits);
6838 return PackedOp;
6839}
6840
6841// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6842SDValue SystemZTargetLowering::
6843lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6844 SDValue PackedOp = Op.getOperand(i: 0);
6845 SDLoc DL(Op);
6846 EVT OutVT = Op.getValueType();
6847 EVT InVT = PackedOp.getValueType();
6848 unsigned InNumElts = InVT.getVectorNumElements();
6849 unsigned OutNumElts = OutVT.getVectorNumElements();
6850 unsigned NumInPerOut = InNumElts / OutNumElts;
6851
6852 SDValue ZeroVec =
6853 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6854
6855 SmallVector<int, 16> Mask(InNumElts);
6856 unsigned ZeroVecElt = InNumElts;
6857 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6858 unsigned MaskElt = PackedElt * NumInPerOut;
6859 unsigned End = MaskElt + NumInPerOut - 1;
6860 for (; MaskElt < End; MaskElt++)
6861 Mask[MaskElt] = ZeroVecElt++;
6862 Mask[MaskElt] = PackedElt;
6863 }
6864 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6865 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6866}
6867
6868SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6869 unsigned ByScalar) const {
6870 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6871 SDValue Op0 = Op.getOperand(i: 0);
6872 SDValue Op1 = Op.getOperand(i: 1);
6873 SDLoc DL(Op);
6874 EVT VT = Op.getValueType();
6875 unsigned ElemBitSize = VT.getScalarSizeInBits();
6876
6877 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6878 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6879 APInt SplatBits, SplatUndef;
6880 unsigned SplatBitSize;
6881 bool HasAnyUndefs;
6882 // Check for constant splats. Use ElemBitSize as the minimum element
6883 // width and reject splats that need wider elements.
6884 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6885 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6886 SplatBitSize == ElemBitSize) {
6887 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6888 DL, VT: MVT::i32);
6889 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6890 }
6891 // Check for variable splats.
6892 BitVector UndefElements;
6893 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
6894 if (Splat) {
6895 // Since i32 is the smallest legal type, we either need a no-op
6896 // or a truncation.
6897 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
6898 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6899 }
6900 }
6901
6902 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6903 // and the shift amount is directly available in a GPR.
6904 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
6905 if (VSN->isSplat()) {
6906 SDValue VSNOp0 = VSN->getOperand(Num: 0);
6907 unsigned Index = VSN->getSplatIndex();
6908 assert(Index < VT.getVectorNumElements() &&
6909 "Splat index should be defined and in first operand");
6910 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6911 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6912 // Since i32 is the smallest legal type, we either need a no-op
6913 // or a truncation.
6914 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
6915 Operand: VSNOp0.getOperand(i: Index));
6916 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6917 }
6918 }
6919 }
6920
6921 // Otherwise just treat the current form as legal.
6922 return Op;
6923}
6924
6925SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
6926 SDLoc DL(Op);
6927
6928 // i128 FSHL with a constant amount that is a multiple of 8 can be
6929 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6930 // facility, FSHL with a constant amount less than 8 can be implemented
6931 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
6932 // combination of the two.
6933 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6934 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6935 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6936 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6937 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6938 if (ShiftAmt > 120) {
6939 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
6940 // SHR_DOUBLE_BIT emits fewer instructions.
6941 SDValue Val =
6942 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6943 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6944 return DAG.getBitcast(VT: MVT::i128, V: Val);
6945 }
6946 SmallVector<int, 16> Mask(16);
6947 for (unsigned Elt = 0; Elt < 16; Elt++)
6948 Mask[Elt] = (ShiftAmt >> 3) + Elt;
6949 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6950 if ((ShiftAmt & 7) == 0)
6951 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6952 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
6953 SDValue Val =
6954 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
6955 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6956 return DAG.getBitcast(VT: MVT::i128, V: Val);
6957 }
6958 }
6959
6960 return SDValue();
6961}
6962
6963SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
6964 SDLoc DL(Op);
6965
6966 // i128 FSHR with a constant amount that is a multiple of 8 can be
6967 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6968 // facility, FSHR with a constant amount less than 8 can be implemented
6969 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
6970 // combination of the two.
6971 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6972 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6973 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6974 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6975 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6976 if (ShiftAmt > 120) {
6977 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
6978 // SHL_DOUBLE_BIT emits fewer instructions.
6979 SDValue Val =
6980 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6981 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6982 return DAG.getBitcast(VT: MVT::i128, V: Val);
6983 }
6984 SmallVector<int, 16> Mask(16);
6985 for (unsigned Elt = 0; Elt < 16; Elt++)
6986 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
6987 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6988 if ((ShiftAmt & 7) == 0)
6989 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6990 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
6991 SDValue Val =
6992 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
6993 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6994 return DAG.getBitcast(VT: MVT::i128, V: Val);
6995 }
6996 }
6997
6998 return SDValue();
6999}
7000
7001static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
7002 SDLoc DL(Op);
7003 SDValue Src = Op.getOperand(i: 0);
7004 MVT DstVT = Op.getSimpleValueType();
7005
7006 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
7007 unsigned SrcAS = N->getSrcAddressSpace();
7008
7009 assert(SrcAS != N->getDestAddressSpace() &&
7010 "addrspacecast must be between different address spaces");
7011
7012 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
7013 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
7014 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
7015 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
7016 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7017 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7018 } else if (DstVT == MVT::i32) {
7019 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
7020 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
7021 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7022 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7023 } else {
7024 report_fatal_error(reason: "Bad address space in addrspacecast");
7025 }
7026 return Op;
7027}
7028
7029SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
7030 SelectionDAG &DAG) const {
7031 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
7032 if (In.getSimpleValueType() != MVT::f16)
7033 return Op; // Legal
7034 return SDValue(); // Let legalizer emit the libcall.
7035}
7036
7037SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
7038 MVT VT, SDValue Arg, SDLoc DL,
7039 SDValue Chain, bool IsStrict) const {
7040 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
7041 MakeLibCallOptions CallOptions;
7042 SDValue Result;
7043 std::tie(args&: Result, args&: Chain) =
7044 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
7045 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
7046}
7047
7048SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
7049 SelectionDAG &DAG) const {
7050 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
7051 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
7052 bool IsStrict = Op->isStrictFPOpcode();
7053 SDLoc DL(Op);
7054 MVT VT = Op.getSimpleValueType();
7055 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7056 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7057 EVT InVT = InOp.getValueType();
7058
7059 // FP to unsigned is not directly supported on z10. Promoting an i32
7060 // result to (signed) i64 doesn't generate an inexact condition (fp
7061 // exception) for values that are outside the i32 range but in the i64
7062 // range, so use the default expansion.
7063 if (!Subtarget.hasFPExtension() && !IsSigned)
7064 // Expand i32/i64. F16 values will be recognized to fit and extended.
7065 return SDValue();
7066
7067 // Conversion from f16 is done via f32.
7068 if (InOp.getSimpleValueType() == MVT::f16) {
7069 SmallVector<SDValue, 2> Results;
7070 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7071 return DAG.getMergeValues(Ops: Results, dl: DL);
7072 }
7073
7074 if (VT == MVT::i128) {
7075 RTLIB::Libcall LC =
7076 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
7077 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7078 }
7079
7080 return Op; // Legal
7081}
7082
7083SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7084 SelectionDAG &DAG) const {
7085 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7086 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7087 bool IsStrict = Op->isStrictFPOpcode();
7088 SDLoc DL(Op);
7089 MVT VT = Op.getSimpleValueType();
7090 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7091 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7092 EVT InVT = InOp.getValueType();
7093
7094 // Conversion to f16 is done via f32.
7095 if (VT == MVT::f16) {
7096 SmallVector<SDValue, 2> Results;
7097 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7098 return DAG.getMergeValues(Ops: Results, dl: DL);
7099 }
7100
7101 // Unsigned to fp is not directly supported on z10.
7102 if (!Subtarget.hasFPExtension() && !IsSigned)
7103 return SDValue(); // Expand i64.
7104
7105 if (InVT == MVT::i128) {
7106 RTLIB::Libcall LC =
7107 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
7108 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7109 }
7110
7111 return Op; // Legal
7112}
7113
7114// Lower an f16 LOAD in case of no vector support.
7115SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7116 SelectionDAG &DAG) const {
7117 EVT RegVT = Op.getValueType();
7118 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7119 (void)RegVT;
7120
7121 // Load as integer.
7122 SDLoc DL(Op);
7123 SDValue NewLd;
7124 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
7125 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7126 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
7127 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
7128 MMO: AtomicLd->getMemOperand());
7129 } else {
7130 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
7131 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7132 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
7133 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
7134 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
7135 }
7136 SDValue F16Val = convertToF16(Op: NewLd, DAG);
7137 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
7138}
7139
7140// Lower an f16 STORE in case of no vector support.
7141SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7142 SelectionDAG &DAG) const {
7143 SDLoc DL(Op);
7144 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
7145
7146 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
7147 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
7148 Ptr: Shft, Val: AtomicSt->getBasePtr(),
7149 MMO: AtomicSt->getMemOperand());
7150
7151 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
7152 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
7153 MMO: St->getMemOperand());
7154}
7155
7156SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7157 SelectionDAG &DAG) const {
7158 SDLoc DL(Op);
7159 MVT ResultVT = Op.getSimpleValueType();
7160 SDValue Arg = Op.getOperand(i: 0);
7161 unsigned Check = Op.getConstantOperandVal(i: 1);
7162
7163 unsigned TDCMask = 0;
7164 if (Check & fcSNan)
7165 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
7166 if (Check & fcQNan)
7167 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
7168 if (Check & fcPosInf)
7169 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
7170 if (Check & fcNegInf)
7171 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
7172 if (Check & fcPosNormal)
7173 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
7174 if (Check & fcNegNormal)
7175 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
7176 if (Check & fcPosSubnormal)
7177 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
7178 if (Check & fcNegSubnormal)
7179 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
7180 if (Check & fcPosZero)
7181 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7182 if (Check & fcNegZero)
7183 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7184 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
7185
7186 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
7187 return getCCResult(DAG, CCReg: Intr);
7188}
7189
7190SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7191 SelectionDAG &DAG) const {
7192 SDLoc DL(Op);
7193 SDValue Chain = Op.getOperand(i: 0);
7194
7195 // STCKF only supports a memory operand, so we have to use a temporary.
7196 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
7197 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7198 MachinePointerInfo MPI =
7199 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7200
7201 // Use STCFK to store the TOD clock into the temporary.
7202 SDValue StoreOps[] = {Chain, StackPtr};
7203 Chain = DAG.getMemIntrinsicNode(
7204 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7205 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7206
7207 // And read it back from there.
7208 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7209}
7210
7211SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7212 SelectionDAG &DAG) const {
7213 switch (Op.getOpcode()) {
7214 case ISD::FRAMEADDR:
7215 return lowerFRAMEADDR(Op, DAG);
7216 case ISD::RETURNADDR:
7217 return lowerRETURNADDR(Op, DAG);
7218 case ISD::BR_CC:
7219 return lowerBR_CC(Op, DAG);
7220 case ISD::SELECT_CC:
7221 return lowerSELECT_CC(Op, DAG);
7222 case ISD::SETCC:
7223 return lowerSETCC(Op, DAG);
7224 case ISD::STRICT_FSETCC:
7225 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7226 case ISD::STRICT_FSETCCS:
7227 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7228 case ISD::GlobalAddress:
7229 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7230 case ISD::GlobalTLSAddress:
7231 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7232 case ISD::BlockAddress:
7233 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7234 case ISD::JumpTable:
7235 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7236 case ISD::ConstantPool:
7237 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7238 case ISD::BITCAST:
7239 return lowerBITCAST(Op, DAG);
7240 case ISD::VASTART:
7241 return lowerVASTART(Op, DAG);
7242 case ISD::VACOPY:
7243 return lowerVACOPY(Op, DAG);
7244 case ISD::DYNAMIC_STACKALLOC:
7245 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7246 case ISD::GET_DYNAMIC_AREA_OFFSET:
7247 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7248 case ISD::MULHS:
7249 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7250 case ISD::MULHU:
7251 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7252 case ISD::SMUL_LOHI:
7253 return lowerSMUL_LOHI(Op, DAG);
7254 case ISD::UMUL_LOHI:
7255 return lowerUMUL_LOHI(Op, DAG);
7256 case ISD::SDIVREM:
7257 return lowerSDIVREM(Op, DAG);
7258 case ISD::UDIVREM:
7259 return lowerUDIVREM(Op, DAG);
7260 case ISD::SADDO:
7261 case ISD::SSUBO:
7262 case ISD::UADDO:
7263 case ISD::USUBO:
7264 return lowerXALUO(Op, DAG);
7265 case ISD::UADDO_CARRY:
7266 case ISD::USUBO_CARRY:
7267 return lowerUADDSUBO_CARRY(Op, DAG);
7268 case ISD::OR:
7269 return lowerOR(Op, DAG);
7270 case ISD::CTPOP:
7271 return lowerCTPOP(Op, DAG);
7272 case ISD::VECREDUCE_ADD:
7273 return lowerVECREDUCE_ADD(Op, DAG);
7274 case ISD::ATOMIC_FENCE:
7275 return lowerATOMIC_FENCE(Op, DAG);
7276 case ISD::ATOMIC_SWAP:
7277 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7278 case ISD::ATOMIC_STORE:
7279 return lowerATOMIC_STORE(Op, DAG);
7280 case ISD::ATOMIC_LOAD:
7281 return lowerATOMIC_LOAD(Op, DAG);
7282 case ISD::ATOMIC_LOAD_ADD:
7283 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7284 case ISD::ATOMIC_LOAD_SUB:
7285 return lowerATOMIC_LOAD_SUB(Op, DAG);
7286 case ISD::ATOMIC_LOAD_AND:
7287 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7288 case ISD::ATOMIC_LOAD_OR:
7289 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7290 case ISD::ATOMIC_LOAD_XOR:
7291 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7292 case ISD::ATOMIC_LOAD_NAND:
7293 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7294 case ISD::ATOMIC_LOAD_MIN:
7295 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7296 case ISD::ATOMIC_LOAD_MAX:
7297 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7298 case ISD::ATOMIC_LOAD_UMIN:
7299 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7300 case ISD::ATOMIC_LOAD_UMAX:
7301 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7302 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7303 return lowerATOMIC_CMP_SWAP(Op, DAG);
7304 case ISD::STACKSAVE:
7305 return lowerSTACKSAVE(Op, DAG);
7306 case ISD::STACKRESTORE:
7307 return lowerSTACKRESTORE(Op, DAG);
7308 case ISD::PREFETCH:
7309 return lowerPREFETCH(Op, DAG);
7310 case ISD::INTRINSIC_W_CHAIN:
7311 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7312 case ISD::INTRINSIC_WO_CHAIN:
7313 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7314 case ISD::BUILD_VECTOR:
7315 return lowerBUILD_VECTOR(Op, DAG);
7316 case ISD::VECTOR_SHUFFLE:
7317 return lowerVECTOR_SHUFFLE(Op, DAG);
7318 case ISD::SCALAR_TO_VECTOR:
7319 return lowerSCALAR_TO_VECTOR(Op, DAG);
7320 case ISD::INSERT_VECTOR_ELT:
7321 return lowerINSERT_VECTOR_ELT(Op, DAG);
7322 case ISD::EXTRACT_VECTOR_ELT:
7323 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7324 case ISD::SIGN_EXTEND_VECTOR_INREG:
7325 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7326 case ISD::ZERO_EXTEND_VECTOR_INREG:
7327 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7328 case ISD::SHL:
7329 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7330 case ISD::SRL:
7331 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7332 case ISD::SRA:
7333 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7334 case ISD::ADDRSPACECAST:
7335 return lowerAddrSpaceCast(Op, DAG);
7336 case ISD::ROTL:
7337 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7338 case ISD::FSHL:
7339 return lowerFSHL(Op, DAG);
7340 case ISD::FSHR:
7341 return lowerFSHR(Op, DAG);
7342 case ISD::FP_EXTEND:
7343 case ISD::STRICT_FP_EXTEND:
7344 return lowerFP_EXTEND(Op, DAG);
7345 case ISD::FP_TO_UINT:
7346 case ISD::FP_TO_SINT:
7347 case ISD::STRICT_FP_TO_UINT:
7348 case ISD::STRICT_FP_TO_SINT:
7349 return lower_FP_TO_INT(Op, DAG);
7350 case ISD::UINT_TO_FP:
7351 case ISD::SINT_TO_FP:
7352 case ISD::STRICT_UINT_TO_FP:
7353 case ISD::STRICT_SINT_TO_FP:
7354 return lower_INT_TO_FP(Op, DAG);
7355 case ISD::LOAD:
7356 return lowerLoadF16(Op, DAG);
7357 case ISD::STORE:
7358 return lowerStoreF16(Op, DAG);
7359 case ISD::IS_FPCLASS:
7360 return lowerIS_FPCLASS(Op, DAG);
7361 case ISD::GET_ROUNDING:
7362 return lowerGET_ROUNDING(Op, DAG);
7363 case ISD::READCYCLECOUNTER:
7364 return lowerREADCYCLECOUNTER(Op, DAG);
7365 case ISD::EH_SJLJ_SETJMP:
7366 case ISD::EH_SJLJ_LONGJMP:
7367 // These operations are legal on our platform, but we cannot actually
7368 // set the operation action to Legal as common code would treat this
7369 // as equivalent to Expand. Instead, we keep the operation action to
7370 // Custom and just leave them unchanged here.
7371 return Op;
7372
7373 default:
7374 llvm_unreachable("Unexpected node to lower");
7375 }
7376}
7377
7378static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7379 const SDLoc &SL) {
7380 // If i128 is legal, just use a normal bitcast.
7381 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7382 return DAG.getBitcast(VT: MVT::f128, V: Src);
7383
7384 // Otherwise, f128 must live in FP128, so do a partwise move.
7385 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7386 &SystemZ::FP128BitRegClass);
7387
7388 SDValue Hi, Lo;
7389 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7390
7391 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7392 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7393
7394 SDNode *Pair = DAG.getMachineNode(
7395 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7396 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7397 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7398 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7399 return SDValue(Pair, 0);
7400}
7401
7402static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7403 const SDLoc &SL) {
7404 // If i128 is legal, just use a normal bitcast.
7405 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7406 return DAG.getBitcast(VT: MVT::i128, V: Src);
7407
7408 // Otherwise, f128 must live in FP128, so do a partwise move.
7409 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7410 &SystemZ::FP128BitRegClass);
7411
7412 SDValue LoFP =
7413 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7414 SDValue HiFP =
7415 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7416 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7417 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7418
7419 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7420}
7421
7422// Lower operations with invalid operand or result types.
7423void
7424SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7425 SmallVectorImpl<SDValue> &Results,
7426 SelectionDAG &DAG) const {
7427 switch (N->getOpcode()) {
7428 case ISD::ATOMIC_LOAD: {
7429 SDLoc DL(N);
7430 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Other);
7431 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7432 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7433 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7434 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7435
7436 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7437 if (N->getValueType(ResNo: 0) == MVT::f128)
7438 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7439 Results.push_back(Elt: Lowered);
7440 Results.push_back(Elt: Res.getValue(R: 1));
7441 break;
7442 }
7443 case ISD::ATOMIC_STORE: {
7444 SDLoc DL(N);
7445 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7446 SDValue Val = N->getOperand(Num: 1);
7447 if (Val.getValueType() == MVT::f128)
7448 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7449 Val = lowerI128ToGR128(DAG, In: Val);
7450
7451 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7452 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7453 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7454 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7455 // We have to enforce sequential consistency by performing a
7456 // serialization operation after the store.
7457 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7458 AtomicOrdering::SequentiallyConsistent)
7459 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7460 VT: MVT::Other, Op1: Res), 0);
7461 Results.push_back(Elt: Res);
7462 break;
7463 }
7464 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7465 SDLoc DL(N);
7466 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::i32, VT3: MVT::Other);
7467 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7468 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7469 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7470 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7471 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7472 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7473 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7474 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7475 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7476 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7477 Results.push_back(Elt: Success);
7478 Results.push_back(Elt: Res.getValue(R: 2));
7479 break;
7480 }
7481 case ISD::BITCAST: {
7482 if (useSoftFloat())
7483 return;
7484 SDLoc DL(N);
7485 SDValue Src = N->getOperand(Num: 0);
7486 EVT SrcVT = Src.getValueType();
7487 EVT ResVT = N->getValueType(ResNo: 0);
7488 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7489 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7490 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7491 if (Subtarget.hasVector()) {
7492 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7493 Results.push_back(Elt: SDValue(
7494 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7495 } else {
7496 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7497 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7498 }
7499 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7500 SDValue ExtractedVal =
7501 Subtarget.hasVector()
7502 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7503 0)
7504 : convertFromF16(Op: Src, DL, DAG);
7505 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7506 }
7507 break;
7508 }
7509 case ISD::UINT_TO_FP:
7510 case ISD::SINT_TO_FP:
7511 case ISD::STRICT_UINT_TO_FP:
7512 case ISD::STRICT_SINT_TO_FP: {
7513 if (useSoftFloat())
7514 return;
7515 bool IsStrict = N->isStrictFPOpcode();
7516 SDLoc DL(N);
7517 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7518 EVT ResVT = N->getValueType(ResNo: 0);
7519 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7520 if (ResVT == MVT::f16) {
7521 if (!IsStrict) {
7522 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7523 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7524 } else {
7525 SDValue OpF32 =
7526 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7527 Ops: {Chain, InOp});
7528 SDValue F16Res;
7529 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7530 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7531 Results.push_back(Elt: F16Res);
7532 Results.push_back(Elt: Chain);
7533 }
7534 }
7535 break;
7536 }
7537 case ISD::FP_TO_UINT:
7538 case ISD::FP_TO_SINT:
7539 case ISD::STRICT_FP_TO_UINT:
7540 case ISD::STRICT_FP_TO_SINT: {
7541 if (useSoftFloat())
7542 return;
7543 bool IsStrict = N->isStrictFPOpcode();
7544 SDLoc DL(N);
7545 EVT ResVT = N->getValueType(ResNo: 0);
7546 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7547 EVT InVT = InOp->getValueType(ResNo: 0);
7548 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7549 if (InVT == MVT::f16) {
7550 if (!IsStrict) {
7551 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7552 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7553 } else {
7554 SDValue InF32;
7555 std::tie(args&: InF32, args&: Chain) =
7556 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7557 SDValue OpF32 =
7558 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7559 Ops: {Chain, InF32});
7560 Results.push_back(Elt: OpF32);
7561 Results.push_back(Elt: OpF32.getValue(R: 1));
7562 }
7563 }
7564 break;
7565 }
7566 default:
7567 llvm_unreachable("Unexpected node to lower");
7568 }
7569}
7570
7571void
7572SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7573 SmallVectorImpl<SDValue> &Results,
7574 SelectionDAG &DAG) const {
7575 return LowerOperationWrapper(N, Results, DAG);
7576}
7577
7578// Return true if VT is a vector whose elements are a whole number of bytes
7579// in width. Also check for presence of vector support.
7580bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7581 if (!Subtarget.hasVector())
7582 return false;
7583
7584 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7585}
7586
7587// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7588// producing a result of type ResVT. Op is a possibly bitcast version
7589// of the input vector and Index is the index (based on type VecVT) that
7590// should be extracted. Return the new extraction if a simplification
7591// was possible or if Force is true.
7592SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7593 EVT VecVT, SDValue Op,
7594 unsigned Index,
7595 DAGCombinerInfo &DCI,
7596 bool Force) const {
7597 SelectionDAG &DAG = DCI.DAG;
7598
7599 // The number of bytes being extracted.
7600 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7601
7602 for (;;) {
7603 unsigned Opcode = Op.getOpcode();
7604 if (Opcode == ISD::BITCAST)
7605 // Look through bitcasts.
7606 Op = Op.getOperand(i: 0);
7607 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7608 canTreatAsByteVector(VT: Op.getValueType())) {
7609 // Get a VPERM-like permute mask and see whether the bytes covered
7610 // by the extracted element are a contiguous sequence from one
7611 // source operand.
7612 SmallVector<int, SystemZ::VectorBytes> Bytes;
7613 if (!getVPermMask(ShuffleOp: Op, Bytes))
7614 break;
7615 int First;
7616 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7617 BytesPerElement, Base&: First))
7618 break;
7619 if (First < 0)
7620 return DAG.getUNDEF(VT: ResVT);
7621 // Make sure the contiguous sequence starts at a multiple of the
7622 // original element size.
7623 unsigned Byte = unsigned(First) % Bytes.size();
7624 if (Byte % BytesPerElement != 0)
7625 break;
7626 // We can get the extracted value directly from an input.
7627 Index = Byte / BytesPerElement;
7628 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7629 Force = true;
7630 } else if (Opcode == ISD::BUILD_VECTOR &&
7631 canTreatAsByteVector(VT: Op.getValueType())) {
7632 // We can only optimize this case if the BUILD_VECTOR elements are
7633 // at least as wide as the extracted value.
7634 EVT OpVT = Op.getValueType();
7635 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7636 if (OpBytesPerElement < BytesPerElement)
7637 break;
7638 // Make sure that the least-significant bit of the extracted value
7639 // is the least significant bit of an input.
7640 unsigned End = (Index + 1) * BytesPerElement;
7641 if (End % OpBytesPerElement != 0)
7642 break;
7643 // We're extracting the low part of one operand of the BUILD_VECTOR.
7644 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7645 EVT ResIntVT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7646 if (!isTypeLegal(VT: ResIntVT))
7647 break;
7648 if (!Op.getValueType().isInteger()) {
7649 EVT OpIntVT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7650 if (!isTypeLegal(VT: OpIntVT))
7651 break;
7652 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OpIntVT, Operand: Op);
7653 DCI.AddToWorklist(N: Op.getNode());
7654 }
7655 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResIntVT, Operand: Op);
7656 if (ResIntVT != ResVT) {
7657 DCI.AddToWorklist(N: Op.getNode());
7658 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7659 }
7660 return Op;
7661 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7662 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7663 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7664 canTreatAsByteVector(VT: Op.getValueType()) &&
7665 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7666 // Make sure that only the unextended bits are significant.
7667 EVT ExtVT = Op.getValueType();
7668 EVT OpVT = Op.getOperand(i: 0).getValueType();
7669 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7670 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7671 unsigned Byte = Index * BytesPerElement;
7672 unsigned SubByte = Byte % ExtBytesPerElement;
7673 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7674 if (SubByte < MinSubByte ||
7675 SubByte + BytesPerElement > ExtBytesPerElement)
7676 break;
7677 // Get the byte offset of the unextended element
7678 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7679 // ...then add the byte offset relative to that element.
7680 Byte += SubByte - MinSubByte;
7681 if (Byte % BytesPerElement != 0)
7682 break;
7683 Op = Op.getOperand(i: 0);
7684 Index = Byte / BytesPerElement;
7685 Force = true;
7686 } else
7687 break;
7688 }
7689 if (Force) {
7690 if (Op.getValueType() != VecVT) {
7691 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7692 DCI.AddToWorklist(N: Op.getNode());
7693 }
7694 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7695 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7696 }
7697 return SDValue();
7698}
7699
7700// Optimize vector operations in scalar value Op on the basis that Op
7701// is truncated to TruncVT.
7702SDValue SystemZTargetLowering::combineTruncateExtract(
7703 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7704 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7705 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7706 // of type TruncVT.
7707 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7708 TruncVT.getSizeInBits() % 8 == 0) {
7709 SDValue Vec = Op.getOperand(i: 0);
7710 EVT VecVT = Vec.getValueType();
7711 if (canTreatAsByteVector(VT: VecVT)) {
7712 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7713 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7714 unsigned TruncBytes = TruncVT.getStoreSize();
7715 if (BytesPerElement % TruncBytes == 0) {
7716 // Calculate the value of Y' in the above description. We are
7717 // splitting the original elements into Scale equal-sized pieces
7718 // and for truncation purposes want the last (least-significant)
7719 // of these pieces for IndexN. This is easiest to do by calculating
7720 // the start index of the following element and then subtracting 1.
7721 unsigned Scale = BytesPerElement / TruncBytes;
7722 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7723
7724 // Defer the creation of the bitcast from X to combineExtract,
7725 // which might be able to optimize the extraction.
7726 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7727 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7728 NumElements: VecVT.getStoreSize() / TruncBytes);
7729 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7730 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7731 }
7732 }
7733 }
7734 }
7735 return SDValue();
7736}
7737
7738SDValue SystemZTargetLowering::combineZERO_EXTEND(
7739 SDNode *N, DAGCombinerInfo &DCI) const {
7740 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7741 SelectionDAG &DAG = DCI.DAG;
7742 SDValue N0 = N->getOperand(Num: 0);
7743 EVT VT = N->getValueType(ResNo: 0);
7744 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7745 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7746 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7747 if (TrueOp && FalseOp) {
7748 SDLoc DL(N0);
7749 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7750 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7751 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7752 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7753 // If N0 has multiple uses, change other uses as well.
7754 if (!N0.hasOneUse()) {
7755 SDValue TruncSelect =
7756 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7757 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7758 }
7759 return NewSelect;
7760 }
7761 }
7762 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7763 // of the result is smaller than the size of X and all the truncated bits
7764 // of X are already zero.
7765 if (N0.getOpcode() == ISD::XOR &&
7766 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7767 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7768 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7769 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7770 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7771 KnownBits Known = DAG.computeKnownBits(Op: X);
7772 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7773 loBit: N0.getValueSizeInBits(),
7774 hiBit: VT.getSizeInBits());
7775 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7776 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7777 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7778 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7779 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7780 }
7781 }
7782 }
7783 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7784 // and VECTOR ADD COMPUTE CARRY for i128:
7785 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7786 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7787 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7788 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7789 // For vector types, these patterns are recognized in the .td file.
7790 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7791 N0.getOperand(i: 0).getValueType() == VT) {
7792 SDValue Op0 = N0.getOperand(i: 0);
7793 SDValue Op1 = N0.getOperand(i: 1);
7794 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
7795 switch (CC) {
7796 case ISD::SETULE:
7797 std::swap(a&: Op0, b&: Op1);
7798 [[fallthrough]];
7799 case ISD::SETUGE:
7800 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7801 case ISD::SETUGT:
7802 std::swap(a&: Op0, b&: Op1);
7803 [[fallthrough]];
7804 case ISD::SETULT:
7805 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7806 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7807 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7808 N2: Op0->getOperand(Num: 1));
7809 break;
7810 default:
7811 break;
7812 }
7813 }
7814
7815 return SDValue();
7816}
7817
7818SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7819 SDNode *N, DAGCombinerInfo &DCI) const {
7820 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7821 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7822 // into (select_cc LHS, RHS, -1, 0, COND)
7823 SelectionDAG &DAG = DCI.DAG;
7824 SDValue N0 = N->getOperand(Num: 0);
7825 EVT VT = N->getValueType(ResNo: 0);
7826 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7827 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7828 N0 = N0.getOperand(i: 0);
7829 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7830 SDLoc DL(N0);
7831 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7832 DAG.getAllOnesConstant(DL, VT),
7833 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7834 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7835 }
7836 return SDValue();
7837}
7838
7839SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7840 SDNode *N, DAGCombinerInfo &DCI) const {
7841 // Convert (sext (ashr (shl X, C1), C2)) to
7842 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7843 // cheap as narrower ones.
7844 SelectionDAG &DAG = DCI.DAG;
7845 SDValue N0 = N->getOperand(Num: 0);
7846 EVT VT = N->getValueType(ResNo: 0);
7847 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7848 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7849 SDValue Inner = N0.getOperand(i: 0);
7850 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7851 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7852 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7853 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7854 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7855 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7856 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7857 Operand: Inner.getOperand(i: 0));
7858 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7859 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7860 VT: ShiftVT));
7861 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7862 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7863 }
7864 }
7865 }
7866
7867 return SDValue();
7868}
7869
7870SDValue SystemZTargetLowering::combineMERGE(
7871 SDNode *N, DAGCombinerInfo &DCI) const {
7872 SelectionDAG &DAG = DCI.DAG;
7873 unsigned Opcode = N->getOpcode();
7874 SDValue Op0 = N->getOperand(Num: 0);
7875 SDValue Op1 = N->getOperand(Num: 1);
7876 if (Op0.getOpcode() == ISD::BITCAST)
7877 Op0 = Op0.getOperand(i: 0);
7878 if (ISD::isBuildVectorAllZeros(N: Op0.getNode())) {
7879 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
7880 // for v4f32.
7881 if (Op1 == N->getOperand(Num: 0))
7882 return Op1;
7883 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
7884 EVT VT = Op1.getValueType();
7885 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
7886 if (ElemBytes <= 4) {
7887 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
7888 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
7889 EVT InVT = VT.changeVectorElementTypeToInteger();
7890 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
7891 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
7892 if (VT != InVT) {
7893 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
7894 DCI.AddToWorklist(N: Op1.getNode());
7895 }
7896 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
7897 DCI.AddToWorklist(N: Op.getNode());
7898 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
7899 }
7900 }
7901 return SDValue();
7902}
7903
7904static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7905 SDNode *&HiPart) {
7906 LoPart = HiPart = nullptr;
7907
7908 // Scan through all users.
7909 for (SDUse &Use : LD->uses()) {
7910 // Skip the uses of the chain.
7911 if (Use.getResNo() != 0)
7912 continue;
7913
7914 // Verify every user is a TRUNCATE to i64 of the low or high half.
7915 SDNode *User = Use.getUser();
7916 bool IsLoPart = true;
7917 if (User->getOpcode() == ISD::SRL &&
7918 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
7919 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
7920 User = *User->user_begin();
7921 IsLoPart = false;
7922 }
7923 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
7924 return false;
7925
7926 if (IsLoPart) {
7927 if (LoPart)
7928 return false;
7929 LoPart = User;
7930 } else {
7931 if (HiPart)
7932 return false;
7933 HiPart = User;
7934 }
7935 }
7936 return true;
7937}
7938
7939static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7940 SDNode *&HiPart) {
7941 LoPart = HiPart = nullptr;
7942
7943 // Scan through all users.
7944 for (SDUse &Use : LD->uses()) {
7945 // Skip the uses of the chain.
7946 if (Use.getResNo() != 0)
7947 continue;
7948
7949 // Verify every user is an EXTRACT_SUBREG of the low or high half.
7950 SDNode *User = Use.getUser();
7951 if (!User->hasOneUse() || !User->isMachineOpcode() ||
7952 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
7953 return false;
7954
7955 switch (User->getConstantOperandVal(Num: 1)) {
7956 case SystemZ::subreg_l64:
7957 if (LoPart)
7958 return false;
7959 LoPart = User;
7960 break;
7961 case SystemZ::subreg_h64:
7962 if (HiPart)
7963 return false;
7964 HiPart = User;
7965 break;
7966 default:
7967 return false;
7968 }
7969 }
7970 return true;
7971}
7972
7973SDValue SystemZTargetLowering::combineLOAD(
7974 SDNode *N, DAGCombinerInfo &DCI) const {
7975 SelectionDAG &DAG = DCI.DAG;
7976 EVT LdVT = N->getValueType(ResNo: 0);
7977 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
7978 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
7979 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7980 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
7981 if (PtrVT != LoadNodeVT) {
7982 SDLoc DL(LN);
7983 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
7984 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
7985 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
7986 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
7987 MMO: LN->getMemOperand());
7988 }
7989 }
7990 }
7991 SDLoc DL(N);
7992
7993 // Replace a 128-bit load that is used solely to move its value into GPRs
7994 // by separate loads of both halves.
7995 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
7996 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
7997 SDNode *LoPart, *HiPart;
7998 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
7999 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
8000 // Rewrite each extraction as an independent load.
8001 SmallVector<SDValue, 2> ArgChains;
8002 if (HiPart) {
8003 SDValue EltLoad = DAG.getLoad(
8004 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
8005 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
8006 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8007
8008 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
8009 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8010 }
8011 if (LoPart) {
8012 SDValue EltLoad = DAG.getLoad(
8013 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
8014 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8015 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
8016 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8017
8018 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
8019 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8020 }
8021
8022 // Collect all chains via TokenFactor.
8023 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
8024 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
8025 DCI.AddToWorklist(N: Chain.getNode());
8026 return SDValue(N, 0);
8027 }
8028 }
8029
8030 if (LdVT.isVector() || LdVT.isInteger())
8031 return SDValue();
8032 // Transform a scalar load that is REPLICATEd as well as having other
8033 // use(s) to the form where the other use(s) use the first element of the
8034 // REPLICATE instead of the load. Otherwise instruction selection will not
8035 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
8036 // point loads.
8037
8038 SDValue Replicate;
8039 SmallVector<SDNode*, 8> OtherUses;
8040 for (SDUse &Use : N->uses()) {
8041 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8042 if (Replicate)
8043 return SDValue(); // Should never happen
8044 Replicate = SDValue(Use.getUser(), 0);
8045 } else if (Use.getResNo() == 0)
8046 OtherUses.push_back(Elt: Use.getUser());
8047 }
8048 if (!Replicate || OtherUses.empty())
8049 return SDValue();
8050
8051 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
8052 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
8053 // Update uses of the loaded Value while preserving old chains.
8054 for (SDNode *U : OtherUses) {
8055 SmallVector<SDValue, 8> Ops;
8056 for (SDValue Op : U->ops())
8057 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
8058 DAG.UpdateNodeOperands(N: U, Ops);
8059 }
8060 return SDValue(N, 0);
8061}
8062
8063bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8064 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8065 return true;
8066 if (Subtarget.hasVectorEnhancements2())
8067 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8068 return true;
8069 return false;
8070}
8071
8072static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8073 if (!VT.isVector() || !VT.isSimple() ||
8074 VT.getSizeInBits() != 128 ||
8075 VT.getScalarSizeInBits() % 8 != 0)
8076 return false;
8077
8078 unsigned NumElts = VT.getVectorNumElements();
8079 for (unsigned i = 0; i < NumElts; ++i) {
8080 if (M[i] < 0) continue; // ignore UNDEF indices
8081 if ((unsigned) M[i] != NumElts - 1 - i)
8082 return false;
8083 }
8084
8085 return true;
8086}
8087
8088static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8089 for (auto *U : StoredVal->users()) {
8090 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8091 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8092 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8093 continue;
8094 } else if (isa<BuildVectorSDNode>(Val: U)) {
8095 SDValue BuildVector = SDValue(U, 0);
8096 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8097 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8098 continue;
8099 }
8100 return false;
8101 }
8102 return true;
8103}
8104
8105static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8106 SDValue &HiPart) {
8107 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8108 return false;
8109
8110 SDValue Op0 = Val.getOperand(i: 0);
8111 SDValue Op1 = Val.getOperand(i: 1);
8112
8113 if (Op0.getOpcode() == ISD::SHL)
8114 std::swap(a&: Op0, b&: Op1);
8115 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8116 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8117 Op1.getConstantOperandVal(i: 1) != 64)
8118 return false;
8119 Op1 = Op1.getOperand(i: 0);
8120
8121 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8122 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8123 return false;
8124 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8125 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8126 return false;
8127
8128 LoPart = Op0.getOperand(i: 0);
8129 HiPart = Op1.getOperand(i: 0);
8130 return true;
8131}
8132
8133static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8134 SDValue &HiPart) {
8135 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8136 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8137 return false;
8138
8139 if (Val->getNumOperands() != 5 ||
8140 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8141 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8142 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8143 return false;
8144
8145 LoPart = Val->getOperand(Num: 1);
8146 HiPart = Val->getOperand(Num: 3);
8147 return true;
8148}
8149
8150SDValue SystemZTargetLowering::combineSTORE(
8151 SDNode *N, DAGCombinerInfo &DCI) const {
8152 SelectionDAG &DAG = DCI.DAG;
8153 auto *SN = cast<StoreSDNode>(Val: N);
8154 auto &Op1 = N->getOperand(Num: 1);
8155 EVT MemVT = SN->getMemoryVT();
8156
8157 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8158 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8159 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8160 if (PtrVT != StoreNodeVT) {
8161 SDLoc DL(SN);
8162 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8163 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8164 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8165 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8166 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8167 }
8168 }
8169
8170 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8171 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8172 // If X has wider elements then convert it to:
8173 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8174 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8175 if (SDValue Value =
8176 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8177 DCI.AddToWorklist(N: Value.getNode());
8178
8179 // Rewrite the store with the new form of stored value.
8180 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8181 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8182 MMO: SN->getMemOperand());
8183 }
8184 }
8185
8186 // combine STORE (LOAD_STACK_GUARD) into MOV_STACKGUARD_DAG
8187 if (Op1->isMachineOpcode() &&
8188 (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) {
8189 // Obtain the frame index the store was targeting.
8190 int FI = cast<FrameIndexSDNode>(Val: SN->getOperand(Num: 2))->getIndex();
8191 // Prepare operands of the MOV_STACKGUARD ISD Node - Chain and FrameIndex.
8192 SDValue Ops[] = {SN->getChain(), DAG.getTargetFrameIndex(FI, VT: MVT::i64)};
8193 return DAG.getNode(Opcode: SystemZISD::MOV_STACKGUARD, DL: SDLoc(SN), VT: MVT::Other, Ops);
8194 }
8195
8196 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8197 if (!SN->isTruncatingStore() &&
8198 Op1.getOpcode() == ISD::BSWAP &&
8199 Op1.getNode()->hasOneUse() &&
8200 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8201
8202 SDValue BSwapOp = Op1.getOperand(i: 0);
8203
8204 if (BSwapOp.getValueType() == MVT::i16)
8205 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8206
8207 SDValue Ops[] = {
8208 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8209 };
8210
8211 return
8212 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8213 Ops, MemVT, MMO: SN->getMemOperand());
8214 }
8215 // Combine STORE (element-swap) into VSTER
8216 if (!SN->isTruncatingStore() &&
8217 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8218 Op1.getNode()->hasOneUse() &&
8219 Subtarget.hasVectorEnhancements2()) {
8220 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8221 ArrayRef<int> ShuffleMask = SVN->getMask();
8222 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8223 SDValue Ops[] = {
8224 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8225 };
8226
8227 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8228 VTList: DAG.getVTList(VT: MVT::Other),
8229 Ops, MemVT, MMO: SN->getMemOperand());
8230 }
8231 }
8232
8233 // Combine STORE (READCYCLECOUNTER) into STCKF.
8234 if (!SN->isTruncatingStore() &&
8235 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8236 Op1.hasOneUse() &&
8237 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8238 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8239 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8240 VTList: DAG.getVTList(VT: MVT::Other),
8241 Ops, MemVT, MMO: SN->getMemOperand());
8242 }
8243
8244 // Transform a store of a 128-bit value moved from parts into two stores.
8245 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8246 SDValue LoPart, HiPart;
8247 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8248 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8249 SDLoc DL(SN);
8250 SDValue Chain0 = DAG.getStore(
8251 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8252 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8253 SDValue Chain1 = DAG.getStore(
8254 Chain: SN->getChain(), dl: DL, Val: LoPart,
8255 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8256 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8257 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8258
8259 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8260 }
8261 }
8262
8263 // Replicate a reg or immediate with VREP instead of scalar multiply or
8264 // immediate load. It seems best to do this during the first DAGCombine as
8265 // it is straight-forward to handle the zero-extend node in the initial
8266 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8267 // extracting an i16 element from a v16i8 vector).
8268 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8269 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8270 SDValue Word = SDValue();
8271 EVT WordVT;
8272
8273 // Find a replicated immediate and return it if found in Word and its
8274 // type in WordVT.
8275 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8276 // Some constants are better handled with a scalar store.
8277 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8278 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8279 return;
8280
8281 APInt Val = C->getAPIntValue();
8282 // Truncate Val in case of a truncating store.
8283 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8284 assert(SN->isTruncatingStore() &&
8285 "Non-truncating store and immediate value does not fit?");
8286 Val = Val.trunc(width: TotBytes * 8);
8287 }
8288
8289 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8290 if (VCI.isVectorConstantLegal(Subtarget) &&
8291 VCI.Opcode == SystemZISD::REPLICATE) {
8292 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8293 WordVT = VCI.VecVT.getScalarType();
8294 }
8295 };
8296
8297 // Find a replicated register and return it if found in Word and its type
8298 // in WordVT.
8299 auto FindReplicatedReg = [&](SDValue MulOp) {
8300 EVT MulVT = MulOp.getValueType();
8301 if (MulOp->getOpcode() == ISD::MUL &&
8302 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8303 // Find a zero extended value and its type.
8304 SDValue LHS = MulOp->getOperand(Num: 0);
8305 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8306 WordVT = LHS->getOperand(Num: 0).getValueType();
8307 else if (LHS->getOpcode() == ISD::AssertZext)
8308 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8309 else
8310 return;
8311 // Find a replicating constant, e.g. 0x00010001.
8312 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8313 SystemZVectorConstantInfo VCI(
8314 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8315 if (VCI.isVectorConstantLegal(Subtarget) &&
8316 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8317 WordVT == VCI.VecVT.getScalarType())
8318 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8319 }
8320 }
8321 };
8322
8323 if (isa<BuildVectorSDNode>(Val: Op1) &&
8324 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8325 SDValue SplatVal = Op1->getOperand(Num: 0);
8326 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8327 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8328 else
8329 FindReplicatedReg(SplatVal);
8330 } else {
8331 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8332 FindReplicatedImm(C, MemVT.getStoreSize());
8333 else
8334 FindReplicatedReg(Op1);
8335 }
8336
8337 if (Word != SDValue()) {
8338 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8339 "Bad type handling");
8340 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8341 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8342 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8343 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8344 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8345 }
8346 }
8347
8348 return SDValue();
8349}
8350
8351SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8352 SDNode *N, DAGCombinerInfo &DCI) const {
8353 SelectionDAG &DAG = DCI.DAG;
8354 // Combine element-swap (LOAD) into VLER
8355 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8356 N->getOperand(Num: 0).hasOneUse() &&
8357 Subtarget.hasVectorEnhancements2()) {
8358 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8359 ArrayRef<int> ShuffleMask = SVN->getMask();
8360 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8361 SDValue Load = N->getOperand(Num: 0);
8362 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8363
8364 // Create the element-swapping load.
8365 SDValue Ops[] = {
8366 LD->getChain(), // Chain
8367 LD->getBasePtr() // Ptr
8368 };
8369 SDValue ESLoad =
8370 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8371 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8372 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8373
8374 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8375 // by the load dead.
8376 DCI.CombineTo(N, Res: ESLoad);
8377
8378 // Next, combine the load away, we give it a bogus result value but a real
8379 // chain result. The result value is dead because the shuffle is dead.
8380 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8381
8382 // Return N so it doesn't get rechecked!
8383 return SDValue(N, 0);
8384 }
8385 }
8386
8387 return SDValue();
8388}
8389
8390SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8391 SDNode *N, DAGCombinerInfo &DCI) const {
8392 SelectionDAG &DAG = DCI.DAG;
8393
8394 if (!Subtarget.hasVector())
8395 return SDValue();
8396
8397 // Look through bitcasts that retain the number of vector elements.
8398 SDValue Op = N->getOperand(Num: 0);
8399 if (Op.getOpcode() == ISD::BITCAST &&
8400 Op.getValueType().isVector() &&
8401 Op.getOperand(i: 0).getValueType().isVector() &&
8402 Op.getValueType().getVectorNumElements() ==
8403 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8404 Op = Op.getOperand(i: 0);
8405
8406 // Pull BSWAP out of a vector extraction.
8407 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8408 EVT VecVT = Op.getValueType();
8409 EVT EltVT = VecVT.getVectorElementType();
8410 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8411 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8412 DCI.AddToWorklist(N: Op.getNode());
8413 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8414 if (EltVT != N->getValueType(ResNo: 0)) {
8415 DCI.AddToWorklist(N: Op.getNode());
8416 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8417 }
8418 return Op;
8419 }
8420
8421 // Try to simplify a vector extraction.
8422 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8423 SDValue Op0 = N->getOperand(Num: 0);
8424 EVT VecVT = Op0.getValueType();
8425 if (canTreatAsByteVector(VT: VecVT))
8426 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8427 Index: IndexN->getZExtValue(), DCI, Force: false);
8428 }
8429 return SDValue();
8430}
8431
8432SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8433 SDNode *N, DAGCombinerInfo &DCI) const {
8434 SelectionDAG &DAG = DCI.DAG;
8435 // (join_dwords X, X) == (replicate X)
8436 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8437 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8438 Operand: N->getOperand(Num: 0));
8439 return SDValue();
8440}
8441
8442static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8443 SDValue Chain1 = N1->getOperand(Num: 0);
8444 SDValue Chain2 = N2->getOperand(Num: 0);
8445
8446 // Trivial case: both nodes take the same chain.
8447 if (Chain1 == Chain2)
8448 return Chain1;
8449
8450 // FIXME - we could handle more complex cases via TokenFactor,
8451 // assuming we can verify that this would not create a cycle.
8452 return SDValue();
8453}
8454
8455SDValue SystemZTargetLowering::combineFP_ROUND(
8456 SDNode *N, DAGCombinerInfo &DCI) const {
8457
8458 if (!Subtarget.hasVector())
8459 return SDValue();
8460
8461 // (fpround (extract_vector_elt X 0))
8462 // (fpround (extract_vector_elt X 1)) ->
8463 // (extract_vector_elt (VROUND X) 0)
8464 // (extract_vector_elt (VROUND X) 2)
8465 //
8466 // This is a special case since the target doesn't really support v2f32s.
8467 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8468 SelectionDAG &DAG = DCI.DAG;
8469 SDValue Op0 = N->getOperand(Num: OpNo);
8470 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8471 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8472 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8473 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8474 Op0.getConstantOperandVal(i: 1) == 0) {
8475 SDValue Vec = Op0.getOperand(i: 0);
8476 for (auto *U : Vec->users()) {
8477 if (U != Op0.getNode() && U->hasOneUse() &&
8478 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8479 U->getOperand(Num: 0) == Vec &&
8480 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8481 U->getConstantOperandVal(Num: 1) == 1) {
8482 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8483 if (OtherRound.getOpcode() == N->getOpcode() &&
8484 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8485 OtherRound.getValueType() == MVT::f32) {
8486 SDValue VRound, Chain;
8487 if (N->isStrictFPOpcode()) {
8488 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8489 if (!Chain)
8490 continue;
8491 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8492 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8493 Chain = VRound.getValue(R: 1);
8494 } else
8495 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8496 VT: MVT::v4f32, Operand: Vec);
8497 DCI.AddToWorklist(N: VRound.getNode());
8498 SDValue Extract1 =
8499 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8500 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8501 DCI.AddToWorklist(N: Extract1.getNode());
8502 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8503 if (Chain)
8504 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8505 SDValue Extract0 =
8506 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8507 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8508 if (Chain)
8509 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8510 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8511 return Extract0;
8512 }
8513 }
8514 }
8515 }
8516 return SDValue();
8517}
8518
8519SDValue SystemZTargetLowering::combineFP_EXTEND(
8520 SDNode *N, DAGCombinerInfo &DCI) const {
8521
8522 if (!Subtarget.hasVector())
8523 return SDValue();
8524
8525 // (fpextend (extract_vector_elt X 0))
8526 // (fpextend (extract_vector_elt X 2)) ->
8527 // (extract_vector_elt (VEXTEND X) 0)
8528 // (extract_vector_elt (VEXTEND X) 1)
8529 //
8530 // This is a special case since the target doesn't really support v2f32s.
8531 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8532 SelectionDAG &DAG = DCI.DAG;
8533 SDValue Op0 = N->getOperand(Num: OpNo);
8534 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8535 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8536 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8537 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8538 Op0.getConstantOperandVal(i: 1) == 0) {
8539 SDValue Vec = Op0.getOperand(i: 0);
8540 for (auto *U : Vec->users()) {
8541 if (U != Op0.getNode() && U->hasOneUse() &&
8542 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8543 U->getOperand(Num: 0) == Vec &&
8544 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8545 U->getConstantOperandVal(Num: 1) == 2) {
8546 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8547 if (OtherExtend.getOpcode() == N->getOpcode() &&
8548 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8549 OtherExtend.getValueType() == MVT::f64) {
8550 SDValue VExtend, Chain;
8551 if (N->isStrictFPOpcode()) {
8552 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8553 if (!Chain)
8554 continue;
8555 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8556 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8557 Chain = VExtend.getValue(R: 1);
8558 } else
8559 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8560 VT: MVT::v2f64, Operand: Vec);
8561 DCI.AddToWorklist(N: VExtend.getNode());
8562 SDValue Extract1 =
8563 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8564 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8565 DCI.AddToWorklist(N: Extract1.getNode());
8566 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8567 if (Chain)
8568 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8569 SDValue Extract0 =
8570 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8571 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8572 if (Chain)
8573 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8574 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8575 return Extract0;
8576 }
8577 }
8578 }
8579 }
8580 return SDValue();
8581}
8582
8583SDValue SystemZTargetLowering::combineINT_TO_FP(
8584 SDNode *N, DAGCombinerInfo &DCI) const {
8585 if (DCI.Level != BeforeLegalizeTypes)
8586 return SDValue();
8587 SelectionDAG &DAG = DCI.DAG;
8588 LLVMContext &Ctx = *DAG.getContext();
8589 unsigned Opcode = N->getOpcode();
8590 EVT OutVT = N->getValueType(ResNo: 0);
8591 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8592 SDValue Op = N->getOperand(Num: 0);
8593 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8594 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8595
8596 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8597 // v2f64 = uint_to_fp v2i16
8598 // =>
8599 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8600 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8601 OutScalarBits <= 64) {
8602 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8603 EVT ExtVT = EVT::getVectorVT(
8604 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8605 unsigned ExtOpcode =
8606 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8607 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8608 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8609 }
8610 return SDValue();
8611}
8612
8613SDValue SystemZTargetLowering::combineFCOPYSIGN(
8614 SDNode *N, DAGCombinerInfo &DCI) const {
8615 SelectionDAG &DAG = DCI.DAG;
8616 EVT VT = N->getValueType(ResNo: 0);
8617 SDValue ValOp = N->getOperand(Num: 0);
8618 SDValue SignOp = N->getOperand(Num: 1);
8619
8620 // Remove the rounding which is not needed.
8621 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8622 SDValue WideOp = SignOp.getOperand(i: 0);
8623 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8624 }
8625
8626 return SDValue();
8627}
8628
8629SDValue SystemZTargetLowering::combineBSWAP(
8630 SDNode *N, DAGCombinerInfo &DCI) const {
8631 SelectionDAG &DAG = DCI.DAG;
8632 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8633 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8634 N->getOperand(Num: 0).hasOneUse() &&
8635 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8636 SDValue Load = N->getOperand(Num: 0);
8637 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8638
8639 // Create the byte-swapping load.
8640 SDValue Ops[] = {
8641 LD->getChain(), // Chain
8642 LD->getBasePtr() // Ptr
8643 };
8644 EVT LoadVT = N->getValueType(ResNo: 0);
8645 if (LoadVT == MVT::i16)
8646 LoadVT = MVT::i32;
8647 SDValue BSLoad =
8648 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8649 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8650 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8651
8652 // If this is an i16 load, insert the truncate.
8653 SDValue ResVal = BSLoad;
8654 if (N->getValueType(ResNo: 0) == MVT::i16)
8655 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8656
8657 // First, combine the bswap away. This makes the value produced by the
8658 // load dead.
8659 DCI.CombineTo(N, Res: ResVal);
8660
8661 // Next, combine the load away, we give it a bogus result value but a real
8662 // chain result. The result value is dead because the bswap is dead.
8663 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8664
8665 // Return N so it doesn't get rechecked!
8666 return SDValue(N, 0);
8667 }
8668
8669 // Look through bitcasts that retain the number of vector elements.
8670 SDValue Op = N->getOperand(Num: 0);
8671 if (Op.getOpcode() == ISD::BITCAST &&
8672 Op.getValueType().isVector() &&
8673 Op.getOperand(i: 0).getValueType().isVector() &&
8674 Op.getValueType().getVectorNumElements() ==
8675 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8676 Op = Op.getOperand(i: 0);
8677
8678 // Push BSWAP into a vector insertion if at least one side then simplifies.
8679 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8680 SDValue Vec = Op.getOperand(i: 0);
8681 SDValue Elt = Op.getOperand(i: 1);
8682 SDValue Idx = Op.getOperand(i: 2);
8683
8684 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8685 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8686 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8687 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8688 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8689 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8690 EVT VecVT = N->getValueType(ResNo: 0);
8691 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8692 if (VecVT != Vec.getValueType()) {
8693 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8694 DCI.AddToWorklist(N: Vec.getNode());
8695 }
8696 if (EltVT != Elt.getValueType()) {
8697 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8698 DCI.AddToWorklist(N: Elt.getNode());
8699 }
8700 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8701 DCI.AddToWorklist(N: Vec.getNode());
8702 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8703 DCI.AddToWorklist(N: Elt.getNode());
8704 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8705 N1: Vec, N2: Elt, N3: Idx);
8706 }
8707 }
8708
8709 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8710 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8711 if (SV && Op.hasOneUse()) {
8712 SDValue Op0 = Op.getOperand(i: 0);
8713 SDValue Op1 = Op.getOperand(i: 1);
8714
8715 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8716 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8717 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8718 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8719 EVT VecVT = N->getValueType(ResNo: 0);
8720 if (VecVT != Op0.getValueType()) {
8721 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8722 DCI.AddToWorklist(N: Op0.getNode());
8723 }
8724 if (VecVT != Op1.getValueType()) {
8725 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8726 DCI.AddToWorklist(N: Op1.getNode());
8727 }
8728 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8729 DCI.AddToWorklist(N: Op0.getNode());
8730 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8731 DCI.AddToWorklist(N: Op1.getNode());
8732 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8733 }
8734 }
8735
8736 return SDValue();
8737}
8738
8739SDValue SystemZTargetLowering::combineSETCC(
8740 SDNode *N, DAGCombinerInfo &DCI) const {
8741 SelectionDAG &DAG = DCI.DAG;
8742 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
8743 const SDValue LHS = N->getOperand(Num: 0);
8744 const SDValue RHS = N->getOperand(Num: 1);
8745 bool CmpNull = isNullConstant(V: RHS);
8746 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8747 EVT VT = N->getValueType(ResNo: 0);
8748 SDLoc DL(N);
8749
8750 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8751 // change the outer compare to a i128 compare. This will normally
8752 // allow the reduction to be recognized in adjustICmp128, and even if
8753 // not, the i128 compare will still generate better code.
8754 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8755 SDValue Src = peekThroughBitcasts(V: LHS);
8756 if (Src.getOpcode() == ISD::SETCC &&
8757 Src.getValueType().isFixedLengthVector() &&
8758 Src.getValueType().getScalarType() == MVT::i1) {
8759 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8760 if (CmpVT.getSizeInBits() == 128) {
8761 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8762 SDValue LHS =
8763 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8764 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8765 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8766 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8767 Flags: N->getFlags());
8768 }
8769 }
8770 }
8771
8772 return SDValue();
8773}
8774
8775static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8776 unsigned Depth = 0) {
8777 // Limit depth of potentially exponential walk.
8778 if (Depth > 5)
8779 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8780
8781 switch (Val.getOpcode()) {
8782 default:
8783 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8784 case SystemZISD::IPM:
8785 if (Val.getOperand(i: 0).getOpcode() == SystemZISD::CLC ||
8786 Val.getOperand(i: 0).getOpcode() == SystemZISD::STRCMP)
8787 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ICMP);
8788 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ANY);
8789 case SystemZISD::SELECT_CCMASK: {
8790 SDValue Op4CCReg = Val.getOperand(i: 4);
8791 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8792 Op4CCReg.getOpcode() == SystemZISD::TM) {
8793 auto [OpCC, OpCCValid] = findCCUse(Val: Op4CCReg.getOperand(i: 0), Depth: Depth + 1);
8794 if (OpCC != SDValue())
8795 return std::make_pair(x&: OpCC, y&: OpCCValid);
8796 }
8797 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8798 if (!CCValid)
8799 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8800 int CCValidVal = CCValid->getZExtValue();
8801 return std::make_pair(x&: Op4CCReg, y&: CCValidVal);
8802 }
8803 case ISD::ADD:
8804 case ISD::AND:
8805 case ISD::OR:
8806 case ISD::XOR:
8807 case ISD::SHL:
8808 case ISD::SRA:
8809 case ISD::SRL:
8810 auto [Op0CC, Op0CCValid] = findCCUse(Val: Val.getOperand(i: 0), Depth: Depth + 1);
8811 if (Op0CC != SDValue())
8812 return std::make_pair(x&: Op0CC, y&: Op0CCValid);
8813 return findCCUse(Val: Val.getOperand(i: 1), Depth: Depth + 1);
8814 }
8815}
8816
8817static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8818 SelectionDAG &DAG);
8819
8820SmallVector<SDValue, 4> static simplifyAssumingCCVal(SDValue &Val, SDValue &CC,
8821 SelectionDAG &DAG) {
8822 SDLoc DL(Val);
8823 auto Opcode = Val.getOpcode();
8824 switch (Opcode) {
8825 default:
8826 return {};
8827 case ISD::Constant:
8828 return {Val, Val, Val, Val};
8829 case SystemZISD::IPM: {
8830 SDValue IPMOp0 = Val.getOperand(i: 0);
8831 if (IPMOp0 != CC)
8832 return {};
8833 SmallVector<SDValue, 4> ShiftedCCVals;
8834 for (auto CC : {0, 1, 2, 3})
8835 ShiftedCCVals.emplace_back(
8836 Args: DAG.getConstant(Val: (CC << SystemZ::IPM_CC), DL, VT: MVT::i32));
8837 return ShiftedCCVals;
8838 }
8839 case SystemZISD::SELECT_CCMASK: {
8840 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8841 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8842 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8843 if (!CCValid || !CCMask)
8844 return {};
8845
8846 int CCValidVal = CCValid->getZExtValue();
8847 int CCMaskVal = CCMask->getZExtValue();
8848 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8849 // recursive call to simplifyAssumingCCVal.
8850 SDValue Op4CCReg = Val.getOperand(i: 4);
8851 if (Op4CCReg != CC)
8852 combineCCMask(CCReg&: Op4CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8853 if (Op4CCReg != CC)
8854 return {};
8855 const auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC, DAG);
8856 const auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC, DAG);
8857 if (TrueSDVals.empty() || FalseSDVals.empty())
8858 return {};
8859 SmallVector<SDValue, 4> MergedSDVals;
8860 for (auto &CCVal : {0, 1, 2, 3})
8861 MergedSDVals.emplace_back(Args: ((CCMaskVal & (1 << (3 - CCVal))) != 0)
8862 ? TrueSDVals[CCVal]
8863 : FalseSDVals[CCVal]);
8864 return MergedSDVals;
8865 }
8866 case ISD::ADD:
8867 case ISD::AND:
8868 case ISD::OR:
8869 case ISD::XOR:
8870 case ISD::SRA:
8871 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8872 // would clobber CC).
8873 if (!Val.hasOneUse())
8874 return {};
8875 [[fallthrough]];
8876 case ISD::SHL:
8877 case ISD::SRL:
8878 SDValue Op0 = Val.getOperand(i: 0), Op1 = Val.getOperand(i: 1);
8879 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC, DAG);
8880 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC, DAG);
8881 if (Op0SDVals.empty() || Op1SDVals.empty())
8882 return {};
8883 SmallVector<SDValue, 4> BinaryOpSDVals;
8884 for (auto CCVal : {0, 1, 2, 3})
8885 BinaryOpSDVals.emplace_back(Args: DAG.getNode(
8886 Opcode, DL, VT: Val.getValueType(), N1: Op0SDVals[CCVal], N2: Op1SDVals[CCVal]));
8887 return BinaryOpSDVals;
8888 }
8889}
8890
8891static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8892 SelectionDAG &DAG) {
8893 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
8894 // set by the CCReg instruction using the CCValid / CCMask masks,
8895 // If the CCReg instruction is itself a ICMP / TM testing the condition
8896 // code set by some other instruction, see whether we can directly
8897 // use that condition code.
8898 auto *CCNode = CCReg.getNode();
8899 if (!CCNode)
8900 return false;
8901
8902 if (CCNode->getOpcode() == SystemZISD::TM) {
8903 if (CCValid != SystemZ::CCMASK_TM)
8904 return false;
8905 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
8906 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8907 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8908 if (!Op0Node || !Op1Node)
8909 return -1;
8910 auto Op0APVal = Op0Node->getAPIntValue();
8911 auto Op1APVal = Op1Node->getAPIntValue();
8912 auto Result = Op0APVal & Op1APVal;
8913 bool AllOnes = Result == Op1APVal;
8914 bool AllZeros = Result == 0;
8915 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
8916 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
8917 };
8918 SDValue Op0 = CCNode->getOperand(Num: 0);
8919 SDValue Op1 = CCNode->getOperand(Num: 1);
8920 auto [Op0CC, Op0CCValid] = findCCUse(Val: Op0);
8921 if (Op0CC == SDValue())
8922 return false;
8923 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC&: Op0CC, DAG);
8924 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC&: Op0CC, DAG);
8925 if (Op0SDVals.empty() || Op1SDVals.empty())
8926 return false;
8927 int NewCCMask = 0;
8928 for (auto CC : {0, 1, 2, 3}) {
8929 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
8930 if (CCVal < 0)
8931 return false;
8932 NewCCMask <<= 1;
8933 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8934 }
8935 NewCCMask &= Op0CCValid;
8936 CCReg = Op0CC;
8937 CCMask = NewCCMask;
8938 CCValid = Op0CCValid;
8939 return true;
8940 }
8941 if (CCNode->getOpcode() != SystemZISD::ICMP ||
8942 CCValid != SystemZ::CCMASK_ICMP)
8943 return false;
8944
8945 SDValue CmpOp0 = CCNode->getOperand(Num: 0);
8946 SDValue CmpOp1 = CCNode->getOperand(Num: 1);
8947 SDValue CmpOp2 = CCNode->getOperand(Num: 2);
8948 auto [Op0CC, Op0CCValid] = findCCUse(Val: CmpOp0);
8949 if (Op0CC != SDValue()) {
8950 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: CmpOp0, CC&: Op0CC, DAG);
8951 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: CmpOp1, CC&: Op0CC, DAG);
8952 if (Op0SDVals.empty() || Op1SDVals.empty())
8953 return false;
8954
8955 auto *CmpType = dyn_cast<ConstantSDNode>(Val&: CmpOp2);
8956 auto CmpTypeVal = CmpType->getZExtValue();
8957 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
8958 const SDValue &Op1Val) {
8959 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8960 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8961 if (!Op0Node || !Op1Node)
8962 return -1;
8963 auto Op0APVal = Op0Node->getAPIntValue();
8964 auto Op1APVal = Op1Node->getAPIntValue();
8965 if (CmpTypeVal == SystemZICMP::SignedOnly)
8966 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(RHS: Op1APVal) ? 1 : 2;
8967 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(RHS: Op1APVal) ? 1 : 2;
8968 };
8969 int NewCCMask = 0;
8970 for (auto CC : {0, 1, 2, 3}) {
8971 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
8972 if (CCVal < 0)
8973 return false;
8974 NewCCMask <<= 1;
8975 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8976 }
8977 NewCCMask &= Op0CCValid;
8978 CCMask = NewCCMask;
8979 CCReg = Op0CC;
8980 CCValid = Op0CCValid;
8981 return true;
8982 }
8983
8984 return false;
8985}
8986
8987// Merging versus split in multiple branches cost.
8988TargetLoweringBase::CondMergingParams
8989SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
8990 const Value *Lhs,
8991 const Value *Rhs,
8992 const Function *) const {
8993 const auto isFlagOutOpCC = [](const Value *V) {
8994 using namespace llvm::PatternMatch;
8995 const Value *RHSVal;
8996 const APInt *RHSC;
8997 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8998 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
8999 if (match(V: I->getOperand(i: 0), P: m_And(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC))) ||
9000 match(V: I, P: m_Cmp(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC)))) {
9001 if (const auto *CB = dyn_cast<CallBase>(Val: RHSVal)) {
9002 if (CB->isInlineAsm()) {
9003 const InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
9004 return IA && IA->getConstraintString().contains(Other: "{@cc}");
9005 }
9006 }
9007 }
9008 }
9009 return false;
9010 };
9011 // Pattern (ICmp %asm) or (ICmp (And %asm)).
9012 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
9013 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
9014 // conditionals will be merged or else conditionals will be split.
9015 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
9016 return {.BaseCost: 3, .LikelyBias: 0, .UnlikelyBias: -1};
9017 // Default.
9018 return {.BaseCost: -1, .LikelyBias: -1, .UnlikelyBias: -1};
9019}
9020
9021SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
9022 DAGCombinerInfo &DCI) const {
9023 SelectionDAG &DAG = DCI.DAG;
9024
9025 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
9026 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9027 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9028 if (!CCValid || !CCMask)
9029 return SDValue();
9030
9031 int CCValidVal = CCValid->getZExtValue();
9032 int CCMaskVal = CCMask->getZExtValue();
9033 SDValue Chain = N->getOperand(Num: 0);
9034 SDValue CCReg = N->getOperand(Num: 4);
9035 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
9036 // the modified BR_CCMASK with the new values.
9037 // In order to avoid conditional branches with full or empty cc masks, do not
9038 // do this if ccmask is 0 or equal to ccvalid.
9039 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG) && CCMaskVal != 0 &&
9040 CCMaskVal != CCValidVal)
9041 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
9042 N1: Chain,
9043 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9044 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
9045 N4: N->getOperand(Num: 3), N5: CCReg);
9046 return SDValue();
9047}
9048
9049SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9050 SDNode *N, DAGCombinerInfo &DCI) const {
9051 SelectionDAG &DAG = DCI.DAG;
9052
9053 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
9054 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9055 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
9056 if (!CCValid || !CCMask)
9057 return SDValue();
9058
9059 int CCValidVal = CCValid->getZExtValue();
9060 int CCMaskVal = CCMask->getZExtValue();
9061 SDValue CCReg = N->getOperand(Num: 4);
9062
9063 bool IsCombinedCCReg = combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
9064
9065 // Populate SDVals vector for each condition code ccval for given Val, which
9066 // can again be another nested select_ccmask with the same CC.
9067 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
9068 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9069 SmallVector<SDValue, 4> Res;
9070 if (Val.getOperand(i: 4) != CCReg)
9071 return SmallVector<SDValue, 4>{};
9072 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
9073 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
9074 if (!CCMask)
9075 return SmallVector<SDValue, 4>{};
9076
9077 int CCMaskVal = CCMask->getZExtValue();
9078 for (auto &CC : {0, 1, 2, 3})
9079 Res.emplace_back(Args&: ((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
9080 : FalseVal);
9081 return Res;
9082 }
9083 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9084 };
9085 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9086 // with CCReg found by combineCCMask or original CCReg.
9087 SDValue TrueVal = N->getOperand(Num: 0);
9088 SDValue FalseVal = N->getOperand(Num: 1);
9089 auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC&: CCReg, DAG);
9090 auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC&: CCReg, DAG);
9091 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9092 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9093 if (TrueSDVals.empty())
9094 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9095 if (FalseSDVals.empty())
9096 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9097 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9098 SmallSet<SDValue, 4> MergedSDValsSet;
9099 // Ignoring CC values outside CCValiid.
9100 for (auto CC : {0, 1, 2, 3}) {
9101 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9102 MergedSDValsSet.insert(V: ((CCMaskVal & (1 << (3 - CC))) != 0)
9103 ? TrueSDVals[CC]
9104 : FalseSDVals[CC]);
9105 }
9106 if (MergedSDValsSet.size() == 1)
9107 return *MergedSDValsSet.begin();
9108 if (MergedSDValsSet.size() == 2) {
9109 auto BeginIt = MergedSDValsSet.begin();
9110 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(x: BeginIt);
9111 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9112 std::swap(a&: NewTrueVal, b&: NewFalseVal);
9113 int NewCCMask = 0;
9114 for (auto CC : {0, 1, 2, 3}) {
9115 NewCCMask <<= 1;
9116 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9117 ? (TrueSDVals[CC] == NewTrueVal)
9118 : (FalseSDVals[CC] == NewTrueVal);
9119 }
9120 CCMaskVal = NewCCMask;
9121 CCMaskVal &= CCValidVal;
9122 TrueVal = NewTrueVal;
9123 FalseVal = NewFalseVal;
9124 IsCombinedCCReg = true;
9125 }
9126 }
9127 // If the condition is trivially false or trivially true after
9128 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9129 // (possibly modified by constructCCSDValsFromSELECT).
9130 if (CCMaskVal == 0)
9131 return FalseVal;
9132 if (CCMaskVal == CCValidVal)
9133 return TrueVal;
9134
9135 if (IsCombinedCCReg)
9136 return DAG.getNode(
9137 Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: TrueVal,
9138 N2: FalseVal, N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9139 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32), N5: CCReg);
9140
9141 return SDValue();
9142}
9143
9144SDValue SystemZTargetLowering::combineGET_CCMASK(
9145 SDNode *N, DAGCombinerInfo &DCI) const {
9146
9147 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9148 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9149 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9150 if (!CCValid || !CCMask)
9151 return SDValue();
9152 int CCValidVal = CCValid->getZExtValue();
9153 int CCMaskVal = CCMask->getZExtValue();
9154
9155 SDValue Select = N->getOperand(Num: 0);
9156 if (Select->getOpcode() == ISD::TRUNCATE)
9157 Select = Select->getOperand(Num: 0);
9158 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9159 return SDValue();
9160
9161 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
9162 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
9163 if (!SelectCCValid || !SelectCCMask)
9164 return SDValue();
9165 int SelectCCValidVal = SelectCCValid->getZExtValue();
9166 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9167
9168 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
9169 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
9170 if (!TrueVal || !FalseVal)
9171 return SDValue();
9172 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9173 ;
9174 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9175 SelectCCMaskVal ^= SelectCCValidVal;
9176 else
9177 return SDValue();
9178
9179 if (SelectCCValidVal & ~CCValidVal)
9180 return SDValue();
9181 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9182 return SDValue();
9183
9184 return Select->getOperand(Num: 4);
9185}
9186
9187SDValue SystemZTargetLowering::combineIntDIVREM(
9188 SDNode *N, DAGCombinerInfo &DCI) const {
9189 SelectionDAG &DAG = DCI.DAG;
9190 EVT VT = N->getValueType(ResNo: 0);
9191 // In the case where the divisor is a vector of constants a cheaper
9192 // sequence of instructions can replace the divide. BuildSDIV is called to
9193 // do this during DAG combining, but it only succeeds when it can build a
9194 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9195 // since it is not Legal but Custom it can only happen before
9196 // legalization. Therefore we must scalarize this early before Combine
9197 // 1. For widened vectors, this is already the result of type legalization.
9198 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9199 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
9200 return DAG.UnrollVectorOp(N);
9201 return SDValue();
9202}
9203
9204
9205// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9206// This is closely modeled after the common-code combineShiftToMULH.
9207SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9208 SDNode *N, DAGCombinerInfo &DCI) const {
9209 SelectionDAG &DAG = DCI.DAG;
9210 SDLoc DL(N);
9211
9212 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9213 "SRL or SRA node is required here!");
9214
9215 if (!Subtarget.hasVector())
9216 return SDValue();
9217
9218 // Check the shift amount. Proceed with the transformation if the shift
9219 // amount is constant.
9220 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
9221 if (!ShiftAmtSrc)
9222 return SDValue();
9223
9224 // The operation feeding into the shift must be an add.
9225 SDValue ShiftOperand = N->getOperand(Num: 0);
9226 if (ShiftOperand.getOpcode() != ISD::ADD)
9227 return SDValue();
9228
9229 // One operand of the add must be a multiply.
9230 SDValue MulOp = ShiftOperand.getOperand(i: 0);
9231 SDValue AddOp = ShiftOperand.getOperand(i: 1);
9232 if (MulOp.getOpcode() != ISD::MUL) {
9233 if (AddOp.getOpcode() != ISD::MUL)
9234 return SDValue();
9235 std::swap(a&: MulOp, b&: AddOp);
9236 }
9237
9238 // All operands must be equivalent extend nodes.
9239 SDValue LeftOp = MulOp.getOperand(i: 0);
9240 SDValue RightOp = MulOp.getOperand(i: 1);
9241
9242 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9243 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9244
9245 if (!IsSignExt && !IsZeroExt)
9246 return SDValue();
9247
9248 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
9249 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9250
9251 SDValue MulhRightOp;
9252 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
9253 unsigned ActiveBits = IsSignExt
9254 ? Constant->getAPIntValue().getSignificantBits()
9255 : Constant->getAPIntValue().getActiveBits();
9256 if (ActiveBits > NarrowVTSize)
9257 return SDValue();
9258 MulhRightOp = DAG.getConstant(
9259 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9260 VT: NarrowVT);
9261 } else {
9262 if (LeftOp.getOpcode() != RightOp.getOpcode())
9263 return SDValue();
9264 // Check that the two extend nodes are the same type.
9265 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
9266 return SDValue();
9267 MulhRightOp = RightOp.getOperand(i: 0);
9268 }
9269
9270 SDValue MulhAddOp;
9271 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
9272 unsigned ActiveBits = IsSignExt
9273 ? Constant->getAPIntValue().getSignificantBits()
9274 : Constant->getAPIntValue().getActiveBits();
9275 if (ActiveBits > NarrowVTSize)
9276 return SDValue();
9277 MulhAddOp = DAG.getConstant(
9278 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9279 VT: NarrowVT);
9280 } else {
9281 if (LeftOp.getOpcode() != AddOp.getOpcode())
9282 return SDValue();
9283 // Check that the two extend nodes are the same type.
9284 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
9285 return SDValue();
9286 MulhAddOp = AddOp.getOperand(i: 0);
9287 }
9288
9289 EVT WideVT = LeftOp.getValueType();
9290 // Proceed with the transformation if the wide types match.
9291 assert((WideVT == RightOp.getValueType()) &&
9292 "Cannot have a multiply node with two different operand types.");
9293 assert((WideVT == AddOp.getValueType()) &&
9294 "Cannot have an add node with two different operand types.");
9295
9296 // Proceed with the transformation if the wide type is twice as large
9297 // as the narrow type.
9298 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9299 return SDValue();
9300
9301 // Check the shift amount with the narrow type size.
9302 // Proceed with the transformation if the shift amount is the width
9303 // of the narrow type.
9304 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9305 if (ShiftAmt != NarrowVTSize)
9306 return SDValue();
9307
9308 // Proceed if we support the multiply-and-add-high operation.
9309 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9310 NarrowVT == MVT::v4i32 ||
9311 (Subtarget.hasVectorEnhancements3() &&
9312 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9313 return SDValue();
9314
9315 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9316 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9317 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9318 N2: MulhRightOp, N3: MulhAddOp);
9319 bool IsSigned = N->getOpcode() == ISD::SRA;
9320 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9321}
9322
9323// Op is an operand of a multiplication. Check whether this can be folded
9324// into an even/odd widening operation; if so, return the opcode to be used
9325// and update Op to the appropriate sub-operand. Note that the caller must
9326// verify that *both* operands of the multiplication support the operation.
9327static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9328 const SystemZSubtarget &Subtarget,
9329 SDValue &Op) {
9330 EVT VT = Op.getValueType();
9331
9332 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9333 // to selecting the even or odd vector elements.
9334 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9335 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9336 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9337 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9338 unsigned NumElts = VT.getVectorNumElements();
9339 Op = Op.getOperand(i: 0);
9340 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9341 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9342 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9343 ArrayRef<int> ShuffleMask = SVN->getMask();
9344 bool CanUseEven = true, CanUseOdd = true;
9345 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9346 if (ShuffleMask[Elt] == -1)
9347 continue;
9348 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9349 CanUseEven = false;
9350 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9351 CanUseOdd = false;
9352 }
9353 Op = Op.getOperand(i: 0);
9354 if (CanUseEven)
9355 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9356 if (CanUseOdd)
9357 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9358 }
9359 }
9360
9361 // For z17, we can also support the v2i64->i128 case, which looks like
9362 // (sign/zero_extend (extract_vector_elt X 0/1))
9363 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9364 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9365 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9366 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9367 Op = Op.getOperand(i: 0);
9368 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9369 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9370 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9371 unsigned Elem = Op.getConstantOperandVal(i: 1);
9372 Op = Op.getOperand(i: 0);
9373 if (Elem == 0)
9374 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9375 if (Elem == 1)
9376 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9377 }
9378 }
9379
9380 return 0;
9381}
9382
9383SDValue SystemZTargetLowering::combineMUL(
9384 SDNode *N, DAGCombinerInfo &DCI) const {
9385 SelectionDAG &DAG = DCI.DAG;
9386
9387 // Detect even/odd widening multiplication.
9388 SDValue Op0 = N->getOperand(Num: 0);
9389 SDValue Op1 = N->getOperand(Num: 1);
9390 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9391 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9392 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9393 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9394
9395 return SDValue();
9396}
9397
9398SDValue SystemZTargetLowering::combineINTRINSIC(
9399 SDNode *N, DAGCombinerInfo &DCI) const {
9400 SelectionDAG &DAG = DCI.DAG;
9401
9402 unsigned Id = N->getConstantOperandVal(Num: 1);
9403 switch (Id) {
9404 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9405 // or larger is simply a vector load.
9406 case Intrinsic::s390_vll:
9407 case Intrinsic::s390_vlrl:
9408 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9409 if (C->getZExtValue() >= 15)
9410 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9411 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9412 break;
9413 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9414 case Intrinsic::s390_vstl:
9415 case Intrinsic::s390_vstrl:
9416 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9417 if (C->getZExtValue() >= 15)
9418 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9419 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9420 break;
9421 }
9422
9423 return SDValue();
9424}
9425
9426SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9427 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9428 return N->getOperand(Num: 0);
9429 return N;
9430}
9431
9432SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9433 DAGCombinerInfo &DCI) const {
9434 switch(N->getOpcode()) {
9435 default: break;
9436 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9437 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9438 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9439 case SystemZISD::MERGE_HIGH:
9440 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9441 case ISD::LOAD: return combineLOAD(N, DCI);
9442 case ISD::STORE: return combineSTORE(N, DCI);
9443 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9444 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9445 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9446 case ISD::STRICT_FP_ROUND:
9447 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9448 case ISD::STRICT_FP_EXTEND:
9449 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9450 case ISD::SINT_TO_FP:
9451 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9452 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9453 case ISD::BSWAP: return combineBSWAP(N, DCI);
9454 case ISD::SETCC: return combineSETCC(N, DCI);
9455 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9456 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9457 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9458 case ISD::SRL:
9459 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9460 case ISD::MUL: return combineMUL(N, DCI);
9461 case ISD::SDIV:
9462 case ISD::UDIV:
9463 case ISD::SREM:
9464 case ISD::UREM: return combineIntDIVREM(N, DCI);
9465 case ISD::INTRINSIC_W_CHAIN:
9466 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9467 }
9468
9469 return SDValue();
9470}
9471
9472// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9473// are for Op.
9474static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9475 unsigned OpNo) {
9476 EVT VT = Op.getValueType();
9477 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9478 APInt SrcDemE;
9479 unsigned Opcode = Op.getOpcode();
9480 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9481 unsigned Id = Op.getConstantOperandVal(i: 0);
9482 switch (Id) {
9483 case Intrinsic::s390_vpksh: // PACKS
9484 case Intrinsic::s390_vpksf:
9485 case Intrinsic::s390_vpksg:
9486 case Intrinsic::s390_vpkshs: // PACKS_CC
9487 case Intrinsic::s390_vpksfs:
9488 case Intrinsic::s390_vpksgs:
9489 case Intrinsic::s390_vpklsh: // PACKLS
9490 case Intrinsic::s390_vpklsf:
9491 case Intrinsic::s390_vpklsg:
9492 case Intrinsic::s390_vpklshs: // PACKLS_CC
9493 case Intrinsic::s390_vpklsfs:
9494 case Intrinsic::s390_vpklsgs:
9495 // VECTOR PACK truncates the elements of two source vectors into one.
9496 SrcDemE = DemandedElts;
9497 if (OpNo == 2)
9498 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9499 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9500 break;
9501 // VECTOR UNPACK extends half the elements of the source vector.
9502 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9503 case Intrinsic::s390_vuphh:
9504 case Intrinsic::s390_vuphf:
9505 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9506 case Intrinsic::s390_vuplhh:
9507 case Intrinsic::s390_vuplhf:
9508 SrcDemE = APInt(NumElts * 2, 0);
9509 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9510 break;
9511 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9512 case Intrinsic::s390_vuplhw:
9513 case Intrinsic::s390_vuplf:
9514 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9515 case Intrinsic::s390_vupllh:
9516 case Intrinsic::s390_vupllf:
9517 SrcDemE = APInt(NumElts * 2, 0);
9518 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9519 break;
9520 case Intrinsic::s390_vpdi: {
9521 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9522 SrcDemE = APInt(NumElts, 0);
9523 if (!DemandedElts[OpNo - 1])
9524 break;
9525 unsigned Mask = Op.getConstantOperandVal(i: 3);
9526 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9527 // Demand input element 0 or 1, given by the mask bit value.
9528 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9529 break;
9530 }
9531 case Intrinsic::s390_vsldb: {
9532 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9533 assert(VT == MVT::v16i8 && "Unexpected type.");
9534 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9535 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9536 unsigned NumSrc0Els = 16 - FirstIdx;
9537 SrcDemE = APInt(NumElts, 0);
9538 if (OpNo == 1) {
9539 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9540 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9541 } else {
9542 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9543 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9544 }
9545 break;
9546 }
9547 case Intrinsic::s390_vperm:
9548 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9549 break;
9550 default:
9551 llvm_unreachable("Unhandled intrinsic.");
9552 break;
9553 }
9554 } else {
9555 switch (Opcode) {
9556 case SystemZISD::JOIN_DWORDS:
9557 // Scalar operand.
9558 SrcDemE = APInt(1, 1);
9559 break;
9560 case SystemZISD::SELECT_CCMASK:
9561 SrcDemE = DemandedElts;
9562 break;
9563 default:
9564 llvm_unreachable("Unhandled opcode.");
9565 break;
9566 }
9567 }
9568 return SrcDemE;
9569}
9570
9571static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9572 const APInt &DemandedElts,
9573 const SelectionDAG &DAG, unsigned Depth,
9574 unsigned OpNo) {
9575 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9576 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9577 KnownBits LHSKnown =
9578 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9579 KnownBits RHSKnown =
9580 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9581 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9582}
9583
9584void
9585SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9586 KnownBits &Known,
9587 const APInt &DemandedElts,
9588 const SelectionDAG &DAG,
9589 unsigned Depth) const {
9590 Known.resetAll();
9591
9592 // Intrinsic CC result is returned in the two low bits.
9593 unsigned Tmp0, Tmp1; // not used
9594 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9595 Known.Zero.setBitsFrom(2);
9596 return;
9597 }
9598 EVT VT = Op.getValueType();
9599 if (Op.getResNo() != 0 || VT == MVT::Untyped)
9600 return;
9601 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9602 "KnownBits does not match VT in bitwidth");
9603 assert ((!VT.isVector() ||
9604 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9605 "DemandedElts does not match VT number of elements");
9606 unsigned BitWidth = Known.getBitWidth();
9607 unsigned Opcode = Op.getOpcode();
9608 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9609 bool IsLogical = false;
9610 unsigned Id = Op.getConstantOperandVal(i: 0);
9611 switch (Id) {
9612 case Intrinsic::s390_vpksh: // PACKS
9613 case Intrinsic::s390_vpksf:
9614 case Intrinsic::s390_vpksg:
9615 case Intrinsic::s390_vpkshs: // PACKS_CC
9616 case Intrinsic::s390_vpksfs:
9617 case Intrinsic::s390_vpksgs:
9618 case Intrinsic::s390_vpklsh: // PACKLS
9619 case Intrinsic::s390_vpklsf:
9620 case Intrinsic::s390_vpklsg:
9621 case Intrinsic::s390_vpklshs: // PACKLS_CC
9622 case Intrinsic::s390_vpklsfs:
9623 case Intrinsic::s390_vpklsgs:
9624 case Intrinsic::s390_vpdi:
9625 case Intrinsic::s390_vsldb:
9626 case Intrinsic::s390_vperm:
9627 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9628 break;
9629 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9630 case Intrinsic::s390_vuplhh:
9631 case Intrinsic::s390_vuplhf:
9632 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9633 case Intrinsic::s390_vupllh:
9634 case Intrinsic::s390_vupllf:
9635 IsLogical = true;
9636 [[fallthrough]];
9637 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9638 case Intrinsic::s390_vuphh:
9639 case Intrinsic::s390_vuphf:
9640 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9641 case Intrinsic::s390_vuplhw:
9642 case Intrinsic::s390_vuplf: {
9643 SDValue SrcOp = Op.getOperand(i: 1);
9644 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9645 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9646 if (IsLogical) {
9647 Known = Known.zext(BitWidth);
9648 } else
9649 Known = Known.sext(BitWidth);
9650 break;
9651 }
9652 default:
9653 break;
9654 }
9655 } else {
9656 switch (Opcode) {
9657 case SystemZISD::JOIN_DWORDS:
9658 case SystemZISD::SELECT_CCMASK:
9659 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9660 break;
9661 case SystemZISD::REPLICATE: {
9662 SDValue SrcOp = Op.getOperand(i: 0);
9663 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9664 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9665 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9666 break;
9667 }
9668 default:
9669 break;
9670 }
9671 }
9672
9673 // Known has the width of the source operand(s). Adjust if needed to match
9674 // the passed bitwidth.
9675 if (Known.getBitWidth() != BitWidth)
9676 Known = Known.anyextOrTrunc(BitWidth);
9677}
9678
9679static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9680 const SelectionDAG &DAG, unsigned Depth,
9681 unsigned OpNo) {
9682 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9683 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9684 if (LHS == 1) return 1; // Early out.
9685 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9686 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9687 if (RHS == 1) return 1; // Early out.
9688 unsigned Common = std::min(a: LHS, b: RHS);
9689 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9690 EVT VT = Op.getValueType();
9691 unsigned VTBits = VT.getScalarSizeInBits();
9692 if (SrcBitWidth > VTBits) { // PACK
9693 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9694 if (Common > SrcExtraBits)
9695 return (Common - SrcExtraBits);
9696 return 1;
9697 }
9698 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9699 return Common;
9700}
9701
9702unsigned
9703SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9704 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9705 unsigned Depth) const {
9706 if (Op.getResNo() != 0)
9707 return 1;
9708 unsigned Opcode = Op.getOpcode();
9709 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9710 unsigned Id = Op.getConstantOperandVal(i: 0);
9711 switch (Id) {
9712 case Intrinsic::s390_vpksh: // PACKS
9713 case Intrinsic::s390_vpksf:
9714 case Intrinsic::s390_vpksg:
9715 case Intrinsic::s390_vpkshs: // PACKS_CC
9716 case Intrinsic::s390_vpksfs:
9717 case Intrinsic::s390_vpksgs:
9718 case Intrinsic::s390_vpklsh: // PACKLS
9719 case Intrinsic::s390_vpklsf:
9720 case Intrinsic::s390_vpklsg:
9721 case Intrinsic::s390_vpklshs: // PACKLS_CC
9722 case Intrinsic::s390_vpklsfs:
9723 case Intrinsic::s390_vpklsgs:
9724 case Intrinsic::s390_vpdi:
9725 case Intrinsic::s390_vsldb:
9726 case Intrinsic::s390_vperm:
9727 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9728 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9729 case Intrinsic::s390_vuphh:
9730 case Intrinsic::s390_vuphf:
9731 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9732 case Intrinsic::s390_vuplhw:
9733 case Intrinsic::s390_vuplf: {
9734 SDValue PackedOp = Op.getOperand(i: 1);
9735 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9736 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9737 EVT VT = Op.getValueType();
9738 unsigned VTBits = VT.getScalarSizeInBits();
9739 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9740 return Tmp;
9741 }
9742 default:
9743 break;
9744 }
9745 } else {
9746 switch (Opcode) {
9747 case SystemZISD::SELECT_CCMASK:
9748 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9749 default:
9750 break;
9751 }
9752 }
9753
9754 return 1;
9755}
9756
9757bool SystemZTargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
9758 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9759 UndefPoisonKind Kind, unsigned Depth) const {
9760 switch (Op->getOpcode()) {
9761 case SystemZISD::PCREL_WRAPPER:
9762 case SystemZISD::PCREL_OFFSET:
9763 return true;
9764 }
9765 return false;
9766}
9767
9768unsigned
9769SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9770 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9771 unsigned StackAlign = TFI->getStackAlignment();
9772 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9773 "Unexpected stack alignment");
9774 // The default stack probe size is 4096 if the function has no
9775 // stack-probe-size attribute.
9776 unsigned StackProbeSize =
9777 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9778 // Round down to the stack alignment.
9779 StackProbeSize &= ~(StackAlign - 1);
9780 return StackProbeSize ? StackProbeSize : StackAlign;
9781}
9782
9783//===----------------------------------------------------------------------===//
9784// Custom insertion
9785//===----------------------------------------------------------------------===//
9786
9787// Force base value Base into a register before MI. Return the register.
9788static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9789 const SystemZInstrInfo *TII) {
9790 MachineBasicBlock *MBB = MI.getParent();
9791 MachineFunction &MF = *MBB->getParent();
9792 MachineRegisterInfo &MRI = MF.getRegInfo();
9793
9794 if (Base.isReg()) {
9795 // Copy Base into a new virtual register to help register coalescing in
9796 // cases with multiple uses.
9797 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9798 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9799 .add(MO: Base);
9800 return Reg;
9801 }
9802
9803 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9804 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9805 .add(MO: Base)
9806 .addImm(Val: 0)
9807 .addReg(RegNo: 0);
9808 return Reg;
9809}
9810
9811// The CC operand of MI might be missing a kill marker because there
9812// were multiple uses of CC, and ISel didn't know which to mark.
9813// Figure out whether MI should have had a kill marker.
9814static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9815 // Scan forward through BB for a use/def of CC.
9816 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9817 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9818 const MachineInstr &MI = *miI;
9819 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9820 return false;
9821 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9822 break; // Should have kill-flag - update below.
9823 }
9824
9825 // If we hit the end of the block, check whether CC is live into a
9826 // successor.
9827 if (miI == MBB->end()) {
9828 for (const MachineBasicBlock *Succ : MBB->successors())
9829 if (Succ->isLiveIn(Reg: SystemZ::CC))
9830 return false;
9831 }
9832
9833 return true;
9834}
9835
9836// Return true if it is OK for this Select pseudo-opcode to be cascaded
9837// together with other Select pseudo-opcodes into a single basic-block with
9838// a conditional jump around it.
9839static bool isSelectPseudo(MachineInstr &MI) {
9840 switch (MI.getOpcode()) {
9841 case SystemZ::Select32:
9842 case SystemZ::Select64:
9843 case SystemZ::Select128:
9844 case SystemZ::SelectF32:
9845 case SystemZ::SelectF64:
9846 case SystemZ::SelectF128:
9847 case SystemZ::SelectVR32:
9848 case SystemZ::SelectVR64:
9849 case SystemZ::SelectVR128:
9850 return true;
9851
9852 default:
9853 return false;
9854 }
9855}
9856
9857// Helper function, which inserts PHI functions into SinkMBB:
9858// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9859// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9860static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9861 MachineBasicBlock *TrueMBB,
9862 MachineBasicBlock *FalseMBB,
9863 MachineBasicBlock *SinkMBB) {
9864 MachineFunction *MF = TrueMBB->getParent();
9865 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9866
9867 MachineInstr *FirstMI = Selects.front();
9868 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9869 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9870
9871 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9872
9873 // As we are creating the PHIs, we have to be careful if there is more than
9874 // one. Later Selects may reference the results of earlier Selects, but later
9875 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9876 // That also means that PHI construction must work forward from earlier to
9877 // later, and that the code must maintain a mapping from earlier PHI's
9878 // destination registers, and the registers that went into the PHI.
9879 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
9880
9881 for (auto *MI : Selects) {
9882 Register DestReg = MI->getOperand(i: 0).getReg();
9883 Register TrueReg = MI->getOperand(i: 1).getReg();
9884 Register FalseReg = MI->getOperand(i: 2).getReg();
9885
9886 // If this Select we are generating is the opposite condition from
9887 // the jump we generated, then we have to swap the operands for the
9888 // PHI that is going to be generated.
9889 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
9890 std::swap(a&: TrueReg, b&: FalseReg);
9891
9892 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
9893 TrueReg = It->second.first;
9894
9895 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
9896 FalseReg = It->second.second;
9897
9898 DebugLoc DL = MI->getDebugLoc();
9899 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
9900 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
9901 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
9902
9903 // Add this PHI to the rewrite table.
9904 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
9905 }
9906
9907 MF->getProperties().resetNoPHIs();
9908}
9909
9910MachineBasicBlock *
9911SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
9912 MachineBasicBlock *BB) const {
9913 MachineFunction &MF = *BB->getParent();
9914 MachineFrameInfo &MFI = MF.getFrameInfo();
9915 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
9916 assert(TFL->hasReservedCallFrame(MF) &&
9917 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
9918 (void)TFL;
9919 // Get the MaxCallFrameSize value and erase MI since it serves no further
9920 // purpose as the call frame is statically reserved in the prolog. Set
9921 // AdjustsStack as MI is *not* mapped as a frame instruction.
9922 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
9923 if (NumBytes > MFI.getMaxCallFrameSize())
9924 MFI.setMaxCallFrameSize(NumBytes);
9925 MFI.setAdjustsStack(true);
9926
9927 MI.eraseFromParent();
9928 return BB;
9929}
9930
9931// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
9932MachineBasicBlock *
9933SystemZTargetLowering::emitSelect(MachineInstr &MI,
9934 MachineBasicBlock *MBB) const {
9935 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
9936 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9937
9938 unsigned CCValid = MI.getOperand(i: 3).getImm();
9939 unsigned CCMask = MI.getOperand(i: 4).getImm();
9940
9941 // If we have a sequence of Select* pseudo instructions using the
9942 // same condition code value, we want to expand all of them into
9943 // a single pair of basic blocks using the same condition.
9944 SmallVector<MachineInstr*, 8> Selects;
9945 SmallVector<MachineInstr*, 8> DbgValues;
9946 Selects.push_back(Elt: &MI);
9947 unsigned Count = 0;
9948 for (MachineInstr &NextMI : llvm::make_range(
9949 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
9950 if (isSelectPseudo(MI&: NextMI)) {
9951 assert(NextMI.getOperand(3).getImm() == CCValid &&
9952 "Bad CCValid operands since CC was not redefined.");
9953 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
9954 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
9955 Selects.push_back(Elt: &NextMI);
9956 continue;
9957 }
9958 break;
9959 }
9960 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9961 NextMI.usesCustomInsertionHook())
9962 break;
9963 bool User = false;
9964 for (auto *SelMI : Selects)
9965 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
9966 User = true;
9967 break;
9968 }
9969 if (NextMI.isDebugInstr()) {
9970 if (User) {
9971 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
9972 DbgValues.push_back(Elt: &NextMI);
9973 }
9974 } else if (User || ++Count > 20)
9975 break;
9976 }
9977
9978 MachineInstr *LastMI = Selects.back();
9979 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9980 checkCCKill(MI&: *LastMI, MBB));
9981 MachineBasicBlock *StartMBB = MBB;
9982 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
9983 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9984
9985 // Unless CC was killed in the last Select instruction, mark it as
9986 // live-in to both FalseMBB and JoinMBB.
9987 if (!CCKilled) {
9988 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
9989 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9990 }
9991
9992 // StartMBB:
9993 // BRC CCMask, JoinMBB
9994 // # fallthrough to FalseMBB
9995 MBB = StartMBB;
9996 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
9997 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
9998 MBB->addSuccessor(Succ: JoinMBB);
9999 MBB->addSuccessor(Succ: FalseMBB);
10000
10001 // FalseMBB:
10002 // # fallthrough to JoinMBB
10003 MBB = FalseMBB;
10004 MBB->addSuccessor(Succ: JoinMBB);
10005
10006 // JoinMBB:
10007 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
10008 // ...
10009 MBB = JoinMBB;
10010 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
10011 for (auto *SelMI : Selects)
10012 SelMI->eraseFromParent();
10013
10014 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
10015 for (auto *DbgMI : DbgValues)
10016 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
10017
10018 return JoinMBB;
10019}
10020
10021// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
10022// StoreOpcode is the store to use and Invert says whether the store should
10023// happen when the condition is false rather than true. If a STORE ON
10024// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
10025MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
10026 MachineBasicBlock *MBB,
10027 unsigned StoreOpcode,
10028 unsigned STOCOpcode,
10029 bool Invert) const {
10030 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10031
10032 Register SrcReg = MI.getOperand(i: 0).getReg();
10033 MachineOperand Base = MI.getOperand(i: 1);
10034 int64_t Disp = MI.getOperand(i: 2).getImm();
10035 Register IndexReg = MI.getOperand(i: 3).getReg();
10036 unsigned CCValid = MI.getOperand(i: 4).getImm();
10037 unsigned CCMask = MI.getOperand(i: 5).getImm();
10038 DebugLoc DL = MI.getDebugLoc();
10039
10040 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
10041
10042 // ISel pattern matching also adds a load memory operand of the same
10043 // address, so take special care to find the storing memory operand.
10044 MachineMemOperand *MMO = nullptr;
10045 for (auto *I : MI.memoperands())
10046 if (I->isStore()) {
10047 MMO = I;
10048 break;
10049 }
10050
10051 // Use STOCOpcode if possible. We could use different store patterns in
10052 // order to avoid matching the index register, but the performance trade-offs
10053 // might be more complicated in that case.
10054 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10055 if (Invert)
10056 CCMask ^= CCValid;
10057
10058 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
10059 .addReg(RegNo: SrcReg)
10060 .add(MO: Base)
10061 .addImm(Val: Disp)
10062 .addImm(Val: CCValid)
10063 .addImm(Val: CCMask)
10064 .addMemOperand(MMO);
10065
10066 MI.eraseFromParent();
10067 return MBB;
10068 }
10069
10070 // Get the condition needed to branch around the store.
10071 if (!Invert)
10072 CCMask ^= CCValid;
10073
10074 MachineBasicBlock *StartMBB = MBB;
10075 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
10076 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10077
10078 // Unless CC was killed in the CondStore instruction, mark it as
10079 // live-in to both FalseMBB and JoinMBB.
10080 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
10081 !checkCCKill(MI, MBB: JoinMBB)) {
10082 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10083 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10084 }
10085
10086 // StartMBB:
10087 // BRC CCMask, JoinMBB
10088 // # fallthrough to FalseMBB
10089 MBB = StartMBB;
10090 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10091 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10092 MBB->addSuccessor(Succ: JoinMBB);
10093 MBB->addSuccessor(Succ: FalseMBB);
10094
10095 // FalseMBB:
10096 // store %SrcReg, %Disp(%Index,%Base)
10097 // # fallthrough to JoinMBB
10098 MBB = FalseMBB;
10099 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
10100 .addReg(RegNo: SrcReg)
10101 .add(MO: Base)
10102 .addImm(Val: Disp)
10103 .addReg(RegNo: IndexReg)
10104 .addMemOperand(MMO);
10105 MBB->addSuccessor(Succ: JoinMBB);
10106
10107 MI.eraseFromParent();
10108 return JoinMBB;
10109}
10110
10111// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10112MachineBasicBlock *
10113SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10114 MachineBasicBlock *MBB,
10115 bool Unsigned) const {
10116 MachineFunction &MF = *MBB->getParent();
10117 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10118 MachineRegisterInfo &MRI = MF.getRegInfo();
10119
10120 // Synthetic instruction to compare 128-bit values.
10121 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10122 Register Op0 = MI.getOperand(i: 0).getReg();
10123 Register Op1 = MI.getOperand(i: 1).getReg();
10124
10125 MachineBasicBlock *StartMBB = MBB;
10126 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10127 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10128
10129 // StartMBB:
10130 //
10131 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10132 // Swap the inputs to get:
10133 // CC 1 if high(Op0) > high(Op1)
10134 // CC 2 if high(Op0) < high(Op1)
10135 // CC 0 if high(Op0) == high(Op1)
10136 //
10137 // If CC != 0, we'd done, so jump over the next instruction.
10138 //
10139 // VEC[L]G Op1, Op0
10140 // JNE JoinMBB
10141 // # fallthrough to HiEqMBB
10142 MBB = StartMBB;
10143 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10144 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
10145 .addReg(RegNo: Op1).addReg(RegNo: Op0);
10146 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10147 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
10148 MBB->addSuccessor(Succ: JoinMBB);
10149 MBB->addSuccessor(Succ: HiEqMBB);
10150
10151 // HiEqMBB:
10152 //
10153 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10154 // Since we already know the high parts are equal, the CC
10155 // result will only depend on the low parts:
10156 // CC 1 if low(Op0) > low(Op1)
10157 // CC 3 if low(Op0) <= low(Op1)
10158 //
10159 // VCHLGS Tmp, Op0, Op1
10160 // # fallthrough to JoinMBB
10161 MBB = HiEqMBB;
10162 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
10163 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
10164 .addReg(RegNo: Op0).addReg(RegNo: Op1);
10165 MBB->addSuccessor(Succ: JoinMBB);
10166
10167 // Mark CC as live-in to JoinMBB.
10168 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10169
10170 MI.eraseFromParent();
10171 return JoinMBB;
10172}
10173
10174// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10175// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10176// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10177// whether the field should be inverted after performing BinOpcode (e.g. for
10178// NAND).
10179MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10180 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10181 bool Invert) const {
10182 MachineFunction &MF = *MBB->getParent();
10183 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10184 MachineRegisterInfo &MRI = MF.getRegInfo();
10185
10186 // Extract the operands. Base can be a register or a frame index.
10187 // Src2 can be a register or immediate.
10188 Register Dest = MI.getOperand(i: 0).getReg();
10189 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10190 int64_t Disp = MI.getOperand(i: 2).getImm();
10191 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
10192 Register BitShift = MI.getOperand(i: 4).getReg();
10193 Register NegBitShift = MI.getOperand(i: 5).getReg();
10194 unsigned BitSize = MI.getOperand(i: 6).getImm();
10195 DebugLoc DL = MI.getDebugLoc();
10196
10197 // Get the right opcodes for the displacement.
10198 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10199 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10200 assert(LOpcode && CSOpcode && "Displacement out of range");
10201
10202 // Create virtual registers for temporary results.
10203 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10204 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10205 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10206 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10207 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10208
10209 // Insert a basic block for the main loop.
10210 MachineBasicBlock *StartMBB = MBB;
10211 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10212 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10213
10214 // StartMBB:
10215 // ...
10216 // %OrigVal = L Disp(%Base)
10217 // # fall through to LoopMBB
10218 MBB = StartMBB;
10219 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10220 MBB->addSuccessor(Succ: LoopMBB);
10221
10222 // LoopMBB:
10223 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10224 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10225 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10226 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10227 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10228 // JNE LoopMBB
10229 // # fall through to DoneMBB
10230 MBB = LoopMBB;
10231 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10232 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10233 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
10234 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10235 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10236 if (Invert) {
10237 // Perform the operation normally and then invert every bit of the field.
10238 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10239 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp)
10240 .addReg(RegNo: RotatedOldVal)
10241 .add(MO: Src2)
10242 .setOperandDead(3);
10243 // XILF with the upper BitSize bits set.
10244 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
10245 .addReg(RegNo: Tmp)
10246 .addImm(Val: -1U << (32 - BitSize))
10247 .setOperandDead(3);
10248 } else if (BinOpcode)
10249 // A simply binary operation.
10250 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
10251 .addReg(RegNo: RotatedOldVal)
10252 .add(MO: Src2)
10253 .setOperandDead(3);
10254 else
10255 // Use RISBG to rotate Src2 into position and use it to replace the
10256 // field in RotatedOldVal.
10257 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
10258 .addReg(RegNo: RotatedOldVal)
10259 .addReg(RegNo: Src2.getReg())
10260 .addImm(Val: 32)
10261 .addImm(Val: 31 + BitSize)
10262 .addImm(Val: 32 - BitSize)
10263 .setOperandDead(6);
10264 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10265 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10266 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10267 .addReg(RegNo: OldVal)
10268 .addReg(RegNo: NewVal)
10269 .add(MO: Base)
10270 .addImm(Val: Disp);
10271 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10272 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10273 MBB->addSuccessor(Succ: LoopMBB);
10274 MBB->addSuccessor(Succ: DoneMBB);
10275
10276 MI.eraseFromParent();
10277 return DoneMBB;
10278}
10279
10280// Implement EmitInstrWithCustomInserter for subword pseudo
10281// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10282// instruction that should be used to compare the current field with the
10283// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10284// for when the current field should be kept.
10285MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10286 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10287 unsigned KeepOldMask) const {
10288 MachineFunction &MF = *MBB->getParent();
10289 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10290 MachineRegisterInfo &MRI = MF.getRegInfo();
10291
10292 // Extract the operands. Base can be a register or a frame index.
10293 Register Dest = MI.getOperand(i: 0).getReg();
10294 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10295 int64_t Disp = MI.getOperand(i: 2).getImm();
10296 Register Src2 = MI.getOperand(i: 3).getReg();
10297 Register BitShift = MI.getOperand(i: 4).getReg();
10298 Register NegBitShift = MI.getOperand(i: 5).getReg();
10299 unsigned BitSize = MI.getOperand(i: 6).getImm();
10300 DebugLoc DL = MI.getDebugLoc();
10301
10302 // Get the right opcodes for the displacement.
10303 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10304 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10305 assert(LOpcode && CSOpcode && "Displacement out of range");
10306
10307 // Create virtual registers for temporary results.
10308 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10309 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10310 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10311 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10312 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10313 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10314
10315 // Insert 3 basic blocks for the loop.
10316 MachineBasicBlock *StartMBB = MBB;
10317 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10318 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10319 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10320 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10321
10322 // StartMBB:
10323 // ...
10324 // %OrigVal = L Disp(%Base)
10325 // # fall through to LoopMBB
10326 MBB = StartMBB;
10327 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10328 MBB->addSuccessor(Succ: LoopMBB);
10329
10330 // LoopMBB:
10331 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10332 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10333 // CompareOpcode %RotatedOldVal, %Src2
10334 // BRC KeepOldMask, UpdateMBB
10335 MBB = LoopMBB;
10336 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10337 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10338 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10339 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10340 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10341 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10342 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10343 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10344 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10345 MBB->addSuccessor(Succ: UpdateMBB);
10346 MBB->addSuccessor(Succ: UseAltMBB);
10347
10348 // UseAltMBB:
10349 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10350 // # fall through to UpdateMBB
10351 MBB = UseAltMBB;
10352 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10353 .addReg(RegNo: RotatedOldVal)
10354 .addReg(RegNo: Src2)
10355 .addImm(Val: 32)
10356 .addImm(Val: 31 + BitSize)
10357 .addImm(Val: 0)
10358 .setOperandDead(6);
10359 MBB->addSuccessor(Succ: UpdateMBB);
10360
10361 // UpdateMBB:
10362 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10363 // [ %RotatedAltVal, UseAltMBB ]
10364 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10365 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10366 // JNE LoopMBB
10367 // # fall through to DoneMBB
10368 MBB = UpdateMBB;
10369 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10370 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10371 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10372 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10373 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10374 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10375 .addReg(RegNo: OldVal)
10376 .addReg(RegNo: NewVal)
10377 .add(MO: Base)
10378 .addImm(Val: Disp);
10379 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10380 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10381 MBB->addSuccessor(Succ: LoopMBB);
10382 MBB->addSuccessor(Succ: DoneMBB);
10383
10384 MI.eraseFromParent();
10385 return DoneMBB;
10386}
10387
10388// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10389// instruction MI.
10390MachineBasicBlock *
10391SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10392 MachineBasicBlock *MBB) const {
10393 MachineFunction &MF = *MBB->getParent();
10394 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10395 MachineRegisterInfo &MRI = MF.getRegInfo();
10396
10397 // Extract the operands. Base can be a register or a frame index.
10398 Register Dest = MI.getOperand(i: 0).getReg();
10399 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10400 int64_t Disp = MI.getOperand(i: 2).getImm();
10401 Register CmpVal = MI.getOperand(i: 3).getReg();
10402 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10403 Register BitShift = MI.getOperand(i: 5).getReg();
10404 Register NegBitShift = MI.getOperand(i: 6).getReg();
10405 int64_t BitSize = MI.getOperand(i: 7).getImm();
10406 DebugLoc DL = MI.getDebugLoc();
10407
10408 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10409
10410 // Get the right opcodes for the displacement and zero-extension.
10411 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10412 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10413 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10414 assert(LOpcode && CSOpcode && "Displacement out of range");
10415
10416 // Create virtual registers for temporary results.
10417 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10418 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10419 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10420 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10421 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10422 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10423 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10424
10425 // Insert 2 basic blocks for the loop.
10426 MachineBasicBlock *StartMBB = MBB;
10427 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10428 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10429 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10430
10431 // StartMBB:
10432 // ...
10433 // %OrigOldVal = L Disp(%Base)
10434 // # fall through to LoopMBB
10435 MBB = StartMBB;
10436 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10437 .add(MO: Base)
10438 .addImm(Val: Disp)
10439 .addReg(RegNo: 0);
10440 MBB->addSuccessor(Succ: LoopMBB);
10441
10442 // LoopMBB:
10443 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10444 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10445 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10446 // ^^ The low BitSize bits contain the field
10447 // of interest.
10448 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10449 // ^^ Replace the upper 32-BitSize bits of the
10450 // swap value with those that we loaded and rotated.
10451 // %Dest = LL[CH] %OldValRot
10452 // CR %Dest, %CmpVal
10453 // JNE DoneMBB
10454 // # Fall through to SetMBB
10455 MBB = LoopMBB;
10456 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10457 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10458 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10459 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10460 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10461 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10462 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10463 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10464 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10465 .addReg(RegNo: SwapVal)
10466 .addReg(RegNo: OldValRot)
10467 .addImm(Val: 32)
10468 .addImm(Val: 63 - BitSize)
10469 .addImm(Val: 0)
10470 .setOperandDead(6);
10471 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10472 .addReg(RegNo: OldValRot);
10473 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10474 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10475 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10476 .addImm(Val: SystemZ::CCMASK_ICMP)
10477 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10478 MBB->addSuccessor(Succ: DoneMBB);
10479 MBB->addSuccessor(Succ: SetMBB);
10480
10481 // SetMBB:
10482 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10483 // ^^ Rotate the new field to its proper position.
10484 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10485 // JNE LoopMBB
10486 // # fall through to ExitMBB
10487 MBB = SetMBB;
10488 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10489 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10490 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10491 .addReg(RegNo: OldVal)
10492 .addReg(RegNo: StoreVal)
10493 .add(MO: Base)
10494 .addImm(Val: Disp);
10495 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10496 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10497 MBB->addSuccessor(Succ: LoopMBB);
10498 MBB->addSuccessor(Succ: DoneMBB);
10499
10500 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10501 // to the block after the loop. At this point, CC may have been defined
10502 // either by the CR in LoopMBB or by the CS in SetMBB.
10503 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10504 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10505
10506 MI.eraseFromParent();
10507 return DoneMBB;
10508}
10509
10510// Emit a move from two GR64s to a GR128.
10511MachineBasicBlock *
10512SystemZTargetLowering::emitPair128(MachineInstr &MI,
10513 MachineBasicBlock *MBB) const {
10514 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10515 const DebugLoc &DL = MI.getDebugLoc();
10516
10517 Register Dest = MI.getOperand(i: 0).getReg();
10518 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest)
10519 .add(MO: MI.getOperand(i: 1))
10520 .addImm(Val: SystemZ::subreg_h64)
10521 .add(MO: MI.getOperand(i: 2))
10522 .addImm(Val: SystemZ::subreg_l64);
10523 MI.eraseFromParent();
10524 return MBB;
10525}
10526
10527// Emit an extension from a GR64 to a GR128. ClearEven is true
10528// if the high register of the GR128 value must be cleared or false if
10529// it's "don't care".
10530MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
10531 MachineBasicBlock *MBB,
10532 bool ClearEven) const {
10533 MachineFunction &MF = *MBB->getParent();
10534 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10535 MachineRegisterInfo &MRI = MF.getRegInfo();
10536 DebugLoc DL = MI.getDebugLoc();
10537
10538 Register Dest = MI.getOperand(i: 0).getReg();
10539 Register Src = MI.getOperand(i: 1).getReg();
10540 Register In128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10541
10542 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: In128);
10543 if (ClearEven) {
10544 Register NewIn128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10545 Register Zero64 = MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10546
10547 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LLILL), DestReg: Zero64)
10548 .addImm(Val: 0);
10549 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewIn128)
10550 .addReg(RegNo: In128).addReg(RegNo: Zero64).addImm(Val: SystemZ::subreg_h64);
10551 In128 = NewIn128;
10552 }
10553 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dest)
10554 .addReg(RegNo: In128).addReg(RegNo: Src).addImm(Val: SystemZ::subreg_l64);
10555
10556 MI.eraseFromParent();
10557 return MBB;
10558}
10559
10560MachineBasicBlock *
10561SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10562 MachineBasicBlock *MBB,
10563 unsigned Opcode, bool IsMemset) const {
10564 MachineFunction &MF = *MBB->getParent();
10565 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10566 MachineRegisterInfo &MRI = MF.getRegInfo();
10567 DebugLoc DL = MI.getDebugLoc();
10568
10569 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10570 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10571 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10572 uint64_t SrcDisp;
10573
10574 // Fold the displacement Disp if it is out of range.
10575 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10576 if (!isUInt<12>(x: Disp)) {
10577 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10578 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10579 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10580 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10581 Base = MachineOperand::CreateReg(Reg, isDef: false);
10582 Disp = 0;
10583 }
10584 };
10585
10586 if (!IsMemset) {
10587 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10588 SrcDisp = MI.getOperand(i: 3).getImm();
10589 } else {
10590 SrcBase = DestBase;
10591 SrcDisp = DestDisp++;
10592 foldDisplIfNeeded(DestBase, DestDisp);
10593 }
10594
10595 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10596 bool IsImmForm = LengthMO.isImm();
10597 bool IsRegForm = !IsImmForm;
10598
10599 // Build and insert one Opcode of Length, with special treatment for memset.
10600 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10601 MachineBasicBlock::iterator InsPos,
10602 MachineOperand DBase, uint64_t DDisp,
10603 MachineOperand SBase, uint64_t SDisp,
10604 unsigned Length) -> void {
10605 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10606 if (IsMemset) {
10607 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10608 if (ByteMO.isImm())
10609 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10610 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10611 else
10612 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10613 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10614 if (--Length == 0)
10615 return;
10616 }
10617 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10618 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10619 .add(MO: SBase).addImm(Val: SDisp)
10620 .setMemRefs(MI.memoperands());
10621 };
10622
10623 bool NeedsLoop = false;
10624 uint64_t ImmLength = 0;
10625 Register LenAdjReg = SystemZ::NoRegister;
10626 if (IsImmForm) {
10627 ImmLength = LengthMO.getImm();
10628 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10629 if (ImmLength == 0) {
10630 MI.eraseFromParent();
10631 return MBB;
10632 }
10633 if (Opcode == SystemZ::CLC) {
10634 if (ImmLength > 3 * 256)
10635 // A two-CLC sequence is a clear win over a loop, not least because
10636 // it needs only one branch. A three-CLC sequence needs the same
10637 // number of branches as a loop (i.e. 2), but is shorter. That
10638 // brings us to lengths greater than 768 bytes. It seems relatively
10639 // likely that a difference will be found within the first 768 bytes,
10640 // so we just optimize for the smallest number of branch
10641 // instructions, in order to avoid polluting the prediction buffer
10642 // too much.
10643 NeedsLoop = true;
10644 } else if (ImmLength > 6 * 256)
10645 // The heuristic we use is to prefer loops for anything that would
10646 // require 7 or more MVCs. With these kinds of sizes there isn't much
10647 // to choose between straight-line code and looping code, since the
10648 // time will be dominated by the MVCs themselves.
10649 NeedsLoop = true;
10650 } else {
10651 NeedsLoop = true;
10652 LenAdjReg = LengthMO.getReg();
10653 }
10654
10655 // When generating more than one CLC, all but the last will need to
10656 // branch to the end when a difference is found.
10657 MachineBasicBlock *EndMBB =
10658 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10659 ? SystemZ::splitBlockAfter(MI, MBB)
10660 : nullptr);
10661
10662 if (NeedsLoop) {
10663 Register StartCountReg =
10664 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10665 if (IsImmForm) {
10666 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10667 ImmLength &= 255;
10668 } else {
10669 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10670 .addReg(RegNo: LenAdjReg)
10671 .addReg(RegNo: 0)
10672 .addImm(Val: 8);
10673 }
10674
10675 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10676 auto loadZeroAddress = [&]() -> MachineOperand {
10677 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10678 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10679 return MachineOperand::CreateReg(Reg, isDef: false);
10680 };
10681 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10682 DestBase = loadZeroAddress();
10683 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10684 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10685
10686 MachineBasicBlock *StartMBB = nullptr;
10687 MachineBasicBlock *LoopMBB = nullptr;
10688 MachineBasicBlock *NextMBB = nullptr;
10689 MachineBasicBlock *DoneMBB = nullptr;
10690 MachineBasicBlock *AllDoneMBB = nullptr;
10691
10692 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10693 Register StartDestReg =
10694 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10695
10696 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10697 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10698 Register ThisDestReg =
10699 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10700 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10701 Register NextDestReg =
10702 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10703 RC = &SystemZ::GR64BitRegClass;
10704 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10705 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10706
10707 if (IsRegForm) {
10708 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10709 StartMBB = SystemZ::emitBlockAfter(MBB);
10710 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10711 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10712 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10713
10714 // MBB:
10715 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10716 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10717 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10718 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10719 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10720 .addMBB(MBB: AllDoneMBB);
10721 MBB->addSuccessor(Succ: AllDoneMBB);
10722 if (!IsMemset)
10723 MBB->addSuccessor(Succ: StartMBB);
10724 else {
10725 // MemsetOneCheckMBB:
10726 // # Jump to MemsetOneMBB for a memset of length 1, or
10727 // # fall thru to StartMBB.
10728 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10729 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10730 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10731 MBB = MemsetOneCheckMBB;
10732 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10733 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10734 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10735 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10736 .addMBB(MBB: MemsetOneMBB);
10737 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10738 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10739
10740 // MemsetOneMBB:
10741 // # Jump back to AllDoneMBB after a single MVI or STC.
10742 MBB = MemsetOneMBB;
10743 insertMemMemOp(MBB, MBB->end(),
10744 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10745 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10746 1);
10747 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10748 MBB->addSuccessor(Succ: AllDoneMBB);
10749 }
10750
10751 // StartMBB:
10752 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10753 MBB = StartMBB;
10754 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10755 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10756 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10757 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10758 .addMBB(MBB: DoneMBB);
10759 MBB->addSuccessor(Succ: DoneMBB);
10760 MBB->addSuccessor(Succ: LoopMBB);
10761 }
10762 else {
10763 StartMBB = MBB;
10764 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10765 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10766 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10767
10768 // StartMBB:
10769 // # fall through to LoopMBB
10770 MBB->addSuccessor(Succ: LoopMBB);
10771
10772 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10773 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10774 if (EndMBB && !ImmLength)
10775 // If the loop handled the whole CLC range, DoneMBB will be empty with
10776 // CC live-through into EndMBB, so add it as live-in.
10777 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10778 }
10779
10780 // LoopMBB:
10781 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10782 // [ %NextDestReg, NextMBB ]
10783 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10784 // [ %NextSrcReg, NextMBB ]
10785 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10786 // [ %NextCountReg, NextMBB ]
10787 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10788 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10789 // ( JLH EndMBB )
10790 //
10791 // The prefetch is used only for MVC. The JLH is used only for CLC.
10792 MBB = LoopMBB;
10793 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10794 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10795 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10796 if (!HaveSingleBase)
10797 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10798 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10799 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10800 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10801 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10802 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10803 if (Opcode == SystemZ::MVC)
10804 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10805 .addImm(Val: SystemZ::PFD_WRITE)
10806 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10807 insertMemMemOp(MBB, MBB->end(),
10808 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10809 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10810 if (EndMBB) {
10811 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10812 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10813 .addMBB(MBB: EndMBB);
10814 MBB->addSuccessor(Succ: EndMBB);
10815 MBB->addSuccessor(Succ: NextMBB);
10816 }
10817
10818 // NextMBB:
10819 // %NextDestReg = LA 256(%ThisDestReg)
10820 // %NextSrcReg = LA 256(%ThisSrcReg)
10821 // %NextCountReg = AGHI %ThisCountReg, -1
10822 // CGHI %NextCountReg, 0
10823 // JLH LoopMBB
10824 // # fall through to DoneMBB
10825 //
10826 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10827 MBB = NextMBB;
10828 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10829 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10830 if (!HaveSingleBase)
10831 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10832 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10833 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10834 .addReg(RegNo: ThisCountReg)
10835 .addImm(Val: -1)
10836 .setOperandDead(3);
10837 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10838 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10839 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10840 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10841 .addMBB(MBB: LoopMBB);
10842 MBB->addSuccessor(Succ: LoopMBB);
10843 MBB->addSuccessor(Succ: DoneMBB);
10844
10845 MBB = DoneMBB;
10846 if (IsRegForm) {
10847 // DoneMBB:
10848 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10849 // # Use EXecute Relative Long for the remainder of the bytes. The target
10850 // instruction of the EXRL will have a length field of 1 since 0 is an
10851 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10852 // 0xff) + 1.
10853 // # Fall through to AllDoneMBB.
10854 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10855 Register RemDestReg = HaveSingleBase ? RemSrcReg
10856 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10857 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10858 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10859 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10860 if (!HaveSingleBase)
10861 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10862 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10863 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10864 if (IsMemset)
10865 insertMemMemOp(MBB, MBB->end(),
10866 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10867 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10868 MachineInstrBuilder EXRL_MIB =
10869 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10870 .addImm(Val: Opcode)
10871 .addReg(RegNo: LenAdjReg)
10872 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10873 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10874 MBB->addSuccessor(Succ: AllDoneMBB);
10875 MBB = AllDoneMBB;
10876 if (Opcode != SystemZ::MVC) {
10877 EXRL_MIB.addReg(RegNo: SystemZ::CC, Flags: RegState::ImplicitDefine);
10878 if (EndMBB)
10879 MBB->addLiveIn(PhysReg: SystemZ::CC);
10880 }
10881 }
10882 MF.getProperties().resetNoPHIs();
10883 }
10884
10885 // Handle any remaining bytes with straight-line code.
10886 while (ImmLength > 0) {
10887 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10888 // The previous iteration might have created out-of-range displacements.
10889 // Apply them using LA/LAY if so.
10890 foldDisplIfNeeded(DestBase, DestDisp);
10891 foldDisplIfNeeded(SrcBase, SrcDisp);
10892 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10893 DestDisp += ThisLength;
10894 SrcDisp += ThisLength;
10895 ImmLength -= ThisLength;
10896 // If there's another CLC to go, branch to the end if a difference
10897 // was found.
10898 if (EndMBB && ImmLength > 0) {
10899 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10900 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10901 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10902 .addMBB(MBB: EndMBB);
10903 MBB->addSuccessor(Succ: EndMBB);
10904 MBB->addSuccessor(Succ: NextMBB);
10905 MBB = NextMBB;
10906 }
10907 }
10908 if (EndMBB) {
10909 MBB->addSuccessor(Succ: EndMBB);
10910 MBB = EndMBB;
10911 MBB->addLiveIn(PhysReg: SystemZ::CC);
10912 }
10913
10914 MI.eraseFromParent();
10915 return MBB;
10916}
10917
10918MachineBasicBlock *
10919SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
10920 MachineBasicBlock *MBB) const {
10921 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10922
10923 DebugLoc DL = MI.getDebugLoc();
10924 MachineOperand DstAddr = earlyUseOperand(Op: MI.getOperand(i: 0));
10925 MachineOperand SrcAddr = earlyUseOperand(Op: MI.getOperand(i: 1));
10926 uint64_t Len = MI.getOperand(i: 2).getImm();
10927 assert(Len > 0 && Len <= 256 && "Memmove of of unsupported constant length.");
10928
10929 // Use MVC or MVCRL after comparing the addresses.
10930 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
10931 MachineBasicBlock *MvcMBB = SystemZ::emitBlockAfter(MBB);
10932 MachineBasicBlock *MvcrlMBB = SystemZ::emitBlockAfter(MBB: MvcMBB);
10933 MBB->addSuccessor(Succ: MvcMBB);
10934 MBB->addSuccessor(Succ: MvcrlMBB);
10935 MvcMBB->addSuccessor(Succ: DoneMBB);
10936 MvcrlMBB->addSuccessor(Succ: DoneMBB);
10937
10938 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGR)).add(MO: SrcAddr).add(MO: DstAddr);
10939 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10940 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
10941 .addMBB(MBB: MvcrlMBB);
10942
10943 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVC))
10944 .add(MO: DstAddr).addImm(Val: 0)
10945 .addImm(Val: Len)
10946 .add(MO: SrcAddr).addImm(Val: 0)
10947 .setMemRefs(MI.memoperands());
10948 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: DoneMBB);
10949
10950 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: SystemZ::R0L).addImm(Val: Len - 1);
10951 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVCRL))
10952 .add(MO: DstAddr).addImm(Val: 0)
10953 .add(MO: SrcAddr).addImm(Val: 0)
10954 .setMemRefs(MI.memoperands());
10955
10956 MI.eraseFromParent();
10957 return DoneMBB;
10958}
10959
10960// Decompose string pseudo-instruction MI into a loop that continually performs
10961// Opcode until CC != 3.
10962MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
10963 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10964 MachineFunction &MF = *MBB->getParent();
10965 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10966 MachineRegisterInfo &MRI = MF.getRegInfo();
10967 DebugLoc DL = MI.getDebugLoc();
10968
10969 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
10970 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
10971 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
10972 uint64_t CharReg = MI.getOperand(i: 3).getReg();
10973
10974 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
10975 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
10976 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
10977 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
10978
10979 MachineBasicBlock *StartMBB = MBB;
10980 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10981 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10982
10983 // StartMBB:
10984 // # fall through to LoopMBB
10985 MBB->addSuccessor(Succ: LoopMBB);
10986
10987 // LoopMBB:
10988 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
10989 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
10990 // R0L = %CharReg
10991 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
10992 // JO LoopMBB
10993 // # fall through to DoneMBB
10994 //
10995 // The load of R0L can be hoisted by post-RA LICM.
10996 MBB = LoopMBB;
10997
10998 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
10999 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
11000 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
11001 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
11002 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
11003 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
11004 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
11005 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
11006 .addReg(RegNo: End1Reg, Flags: RegState::Define).addReg(RegNo: End2Reg, Flags: RegState::Define)
11007 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
11008 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11009 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
11010 MBB->addSuccessor(Succ: LoopMBB);
11011 MBB->addSuccessor(Succ: DoneMBB);
11012
11013 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
11014
11015 MI.eraseFromParent();
11016 return DoneMBB;
11017}
11018
11019// Update TBEGIN instruction with final opcode and register clobbers.
11020MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
11021 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
11022 bool NoFloat) const {
11023 MachineFunction &MF = *MBB->getParent();
11024 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11025 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11026
11027 // Update opcode.
11028 MI.setDesc(TII->get(Opcode));
11029
11030 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
11031 // Make sure to add the corresponding GRSM bits if they are missing.
11032 uint64_t Control = MI.getOperand(i: 2).getImm();
11033 static const unsigned GPRControlBit[16] = {
11034 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
11035 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
11036 };
11037 Control |= GPRControlBit[15];
11038 if (TFI->hasFP(MF))
11039 Control |= GPRControlBit[11];
11040 MI.getOperand(i: 2).setImm(Control);
11041
11042 // Add GPR clobbers.
11043 for (int I = 0; I < 16; I++) {
11044 if ((Control & GPRControlBit[I]) == 0) {
11045 unsigned Reg = SystemZMC::GR64Regs[I];
11046 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11047 }
11048 }
11049
11050 // Add FPR/VR clobbers.
11051 if (!NoFloat && (Control & 4) != 0) {
11052 if (Subtarget.hasVector()) {
11053 for (unsigned Reg : SystemZMC::VR128Regs) {
11054 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11055 }
11056 } else {
11057 for (unsigned Reg : SystemZMC::FP64Regs) {
11058 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11059 }
11060 }
11061 }
11062
11063 return MBB;
11064}
11065
11066MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
11067 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11068 MachineFunction &MF = *MBB->getParent();
11069 MachineRegisterInfo *MRI = &MF.getRegInfo();
11070 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11071 DebugLoc DL = MI.getDebugLoc();
11072
11073 Register SrcReg = MI.getOperand(i: 0).getReg();
11074
11075 // Create new virtual register of the same class as source.
11076 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
11077 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
11078
11079 // Replace pseudo with a normal load-and-test that models the def as
11080 // well.
11081 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
11082 .addReg(RegNo: SrcReg)
11083 .setMIFlags(MI.getFlags());
11084 MI.eraseFromParent();
11085
11086 return MBB;
11087}
11088
11089MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
11090 MachineInstr &MI, MachineBasicBlock *MBB) const {
11091 MachineFunction &MF = *MBB->getParent();
11092 MachineRegisterInfo *MRI = &MF.getRegInfo();
11093 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11094 DebugLoc DL = MI.getDebugLoc();
11095 const unsigned ProbeSize = getStackProbeSize(MF);
11096 Register DstReg = MI.getOperand(i: 0).getReg();
11097 Register SizeReg = MI.getOperand(i: 2).getReg();
11098
11099 MachineBasicBlock *StartMBB = MBB;
11100 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11101 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11102 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
11103 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
11104 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
11105
11106 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
11107 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
11108
11109 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11110 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11111
11112 // LoopTestMBB
11113 // BRC TailTestMBB
11114 // # fallthrough to LoopBodyMBB
11115 StartMBB->addSuccessor(Succ: LoopTestMBB);
11116 MBB = LoopTestMBB;
11117 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
11118 .addReg(RegNo: SizeReg)
11119 .addMBB(MBB: StartMBB)
11120 .addReg(RegNo: IncReg)
11121 .addMBB(MBB: LoopBodyMBB);
11122 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
11123 .addReg(RegNo: PHIReg)
11124 .addImm(Val: ProbeSize);
11125 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11126 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
11127 .addMBB(MBB: TailTestMBB);
11128 MBB->addSuccessor(Succ: LoopBodyMBB);
11129 MBB->addSuccessor(Succ: TailTestMBB);
11130
11131 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
11132 // J LoopTestMBB
11133 MBB = LoopBodyMBB;
11134 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
11135 .addReg(RegNo: PHIReg)
11136 .addImm(Val: ProbeSize)
11137 .setOperandDead(3);
11138 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
11139 .addReg(RegNo: SystemZ::R15D)
11140 .addImm(Val: ProbeSize)
11141 .setOperandDead(3);
11142 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11143 .addReg(RegNo: SystemZ::R15D)
11144 .addReg(RegNo: SystemZ::R15D)
11145 .addImm(Val: ProbeSize - 8)
11146 .addReg(RegNo: 0)
11147 .setOperandDead(4)
11148 .setMemRefs(VolLdMMO);
11149 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
11150 MBB->addSuccessor(Succ: LoopTestMBB);
11151
11152 // TailTestMBB
11153 // BRC DoneMBB
11154 // # fallthrough to TailMBB
11155 MBB = TailTestMBB;
11156 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
11157 .addReg(RegNo: PHIReg)
11158 .addImm(Val: 0);
11159 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11160 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
11161 .addMBB(MBB: DoneMBB);
11162 MBB->addSuccessor(Succ: TailMBB);
11163 MBB->addSuccessor(Succ: DoneMBB);
11164
11165 // TailMBB
11166 // # fallthrough to DoneMBB
11167 MBB = TailMBB;
11168 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
11169 .addReg(RegNo: SystemZ::R15D)
11170 .addReg(RegNo: PHIReg)
11171 .setOperandDead(3);
11172 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11173 .addReg(RegNo: SystemZ::R15D)
11174 .addReg(RegNo: SystemZ::R15D)
11175 .addImm(Val: -8)
11176 .addReg(RegNo: PHIReg)
11177 .setOperandDead(4)
11178 .setMemRefs(VolLdMMO);
11179 MBB->addSuccessor(Succ: DoneMBB);
11180
11181 // DoneMBB
11182 MBB = DoneMBB;
11183 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
11184 .addReg(RegNo: SystemZ::R15D);
11185
11186 MI.eraseFromParent();
11187 return DoneMBB;
11188}
11189
11190SDValue SystemZTargetLowering::
11191getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11192 MachineFunction &MF = DAG.getMachineFunction();
11193 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11194 SDLoc DL(SP);
11195 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
11196 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
11197}
11198
11199// Replace a _STACKGUARD_DAG pseudo with a _STACKGUARD pseudo, adding
11200// a dead early-clobber def reg that will be used as a scratch register
11201// when the pseudo is expanded.
11202MachineBasicBlock *SystemZTargetLowering::emitStackGuardPseudo(
11203 MachineInstr &MI, MachineBasicBlock *MBB, unsigned PseudoOp) const {
11204 MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
11205 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11206 DebugLoc DL = MI.getDebugLoc();
11207 Register AddrReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11208 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PseudoOp), DestReg: AddrReg)
11209 .addFrameIndex(Idx: MI.getOperand(i: 0).getIndex())
11210 .addImm(Val: MI.getOperand(i: 1).getImm());
11211 MI.eraseFromParent();
11212 return MBB;
11213}
11214
11215MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
11216 MachineInstr &MI, MachineBasicBlock *MBB) const {
11217 switch (MI.getOpcode()) {
11218 case SystemZ::ADJCALLSTACKDOWN:
11219 case SystemZ::ADJCALLSTACKUP:
11220 return emitAdjCallStack(MI, BB: MBB);
11221
11222 case SystemZ::Select32:
11223 case SystemZ::Select64:
11224 case SystemZ::Select128:
11225 case SystemZ::SelectF32:
11226 case SystemZ::SelectF64:
11227 case SystemZ::SelectF128:
11228 case SystemZ::SelectVR32:
11229 case SystemZ::SelectVR64:
11230 case SystemZ::SelectVR128:
11231 return emitSelect(MI, MBB);
11232
11233 case SystemZ::CondStore8Mux:
11234 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
11235 case SystemZ::CondStore8MuxInv:
11236 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
11237 case SystemZ::CondStore16Mux:
11238 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
11239 case SystemZ::CondStore16MuxInv:
11240 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
11241 case SystemZ::CondStore32Mux:
11242 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
11243 case SystemZ::CondStore32MuxInv:
11244 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
11245 case SystemZ::CondStore8:
11246 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
11247 case SystemZ::CondStore8Inv:
11248 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
11249 case SystemZ::CondStore16:
11250 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
11251 case SystemZ::CondStore16Inv:
11252 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
11253 case SystemZ::CondStore32:
11254 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
11255 case SystemZ::CondStore32Inv:
11256 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
11257 case SystemZ::CondStore64:
11258 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
11259 case SystemZ::CondStore64Inv:
11260 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
11261 case SystemZ::CondStoreF32:
11262 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
11263 case SystemZ::CondStoreF32Inv:
11264 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
11265 case SystemZ::CondStoreF64:
11266 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
11267 case SystemZ::CondStoreF64Inv:
11268 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
11269
11270 case SystemZ::SCmp128Hi:
11271 return emitICmp128Hi(MI, MBB, Unsigned: false);
11272 case SystemZ::UCmp128Hi:
11273 return emitICmp128Hi(MI, MBB, Unsigned: true);
11274
11275 case SystemZ::PAIR128:
11276 return emitPair128(MI, MBB);
11277 case SystemZ::AEXT128:
11278 return emitExt128(MI, MBB, ClearEven: false);
11279 case SystemZ::ZEXT128:
11280 return emitExt128(MI, MBB, ClearEven: true);
11281
11282 case SystemZ::ATOMIC_SWAPW:
11283 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
11284
11285 case SystemZ::ATOMIC_LOADW_AR:
11286 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
11287 case SystemZ::ATOMIC_LOADW_AFI:
11288 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
11289
11290 case SystemZ::ATOMIC_LOADW_SR:
11291 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
11292
11293 case SystemZ::ATOMIC_LOADW_NR:
11294 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
11295 case SystemZ::ATOMIC_LOADW_NILH:
11296 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
11297
11298 case SystemZ::ATOMIC_LOADW_OR:
11299 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
11300 case SystemZ::ATOMIC_LOADW_OILH:
11301 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
11302
11303 case SystemZ::ATOMIC_LOADW_XR:
11304 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
11305 case SystemZ::ATOMIC_LOADW_XILF:
11306 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
11307
11308 case SystemZ::ATOMIC_LOADW_NRi:
11309 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
11310 case SystemZ::ATOMIC_LOADW_NILHi:
11311 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
11312
11313 case SystemZ::ATOMIC_LOADW_MIN:
11314 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11315 case SystemZ::ATOMIC_LOADW_MAX:
11316 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11317 case SystemZ::ATOMIC_LOADW_UMIN:
11318 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11319 case SystemZ::ATOMIC_LOADW_UMAX:
11320 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11321
11322 case SystemZ::ATOMIC_CMP_SWAPW:
11323 return emitAtomicCmpSwapW(MI, MBB);
11324 case SystemZ::MVCImm:
11325 case SystemZ::MVCReg:
11326 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
11327 case SystemZ::NCImm:
11328 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
11329 case SystemZ::OCImm:
11330 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
11331 case SystemZ::XCImm:
11332 case SystemZ::XCReg:
11333 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
11334 case SystemZ::CLCImm:
11335 case SystemZ::CLCReg:
11336 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
11337 case SystemZ::MemsetImmImm:
11338 case SystemZ::MemsetImmReg:
11339 case SystemZ::MemsetRegImm:
11340 case SystemZ::MemsetRegReg:
11341 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
11342 case SystemZ::MemmoveImm:
11343 return emitMemmoveImm(MI, MBB);
11344 case SystemZ::CLSTLoop:
11345 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
11346 case SystemZ::MVSTLoop:
11347 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
11348 case SystemZ::SRSTLoop:
11349 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
11350 case SystemZ::TBEGIN:
11351 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
11352 case SystemZ::TBEGIN_nofloat:
11353 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
11354 case SystemZ::TBEGINC:
11355 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
11356 case SystemZ::LTEBRCompare_Pseudo:
11357 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
11358 case SystemZ::LTDBRCompare_Pseudo:
11359 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
11360 case SystemZ::LTXBRCompare_Pseudo:
11361 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
11362
11363 case SystemZ::PROBED_ALLOCA:
11364 return emitProbedAlloca(MI, MBB);
11365 case SystemZ::EH_SjLj_SetJmp:
11366 return emitEHSjLjSetJmp(MI, MBB);
11367 case SystemZ::EH_SjLj_LongJmp:
11368 return emitEHSjLjLongJmp(MI, MBB);
11369
11370 case TargetOpcode::STACKMAP:
11371 case TargetOpcode::PATCHPOINT:
11372 return emitPatchPoint(MI, MBB);
11373
11374 case SystemZ::MOV_STACKGUARD_DAG:
11375 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::MOV_STACKGUARD);
11376
11377 case SystemZ::CMP_STACKGUARD_DAG:
11378 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::CMP_STACKGUARD);
11379
11380 default:
11381 llvm_unreachable("Unexpected instr type to insert");
11382 }
11383}
11384
11385// This is only used by the isel schedulers, and is needed only to prevent
11386// compiler from crashing when list-ilp is used.
11387const TargetRegisterClass *
11388SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
11389 if (VT == MVT::Untyped)
11390 return &SystemZ::ADDR128BitRegClass;
11391 return TargetLowering::getRepRegClassFor(VT);
11392}
11393
11394SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11395 SelectionDAG &DAG) const {
11396 SDLoc dl(Op);
11397 /*
11398 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11399 settings:
11400 00 Round to nearest
11401 01 Round to 0
11402 10 Round to +inf
11403 11 Round to -inf
11404
11405 FLT_ROUNDS, on the other hand, expects the following:
11406 -1 Undefined
11407 0 Round to 0
11408 1 Round to nearest
11409 2 Round to +inf
11410 3 Round to -inf
11411 */
11412
11413 // Save FPC to register.
11414 SDValue Chain = Op.getOperand(i: 0);
11415 SDValue EFPC(
11416 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11417 Chain = EFPC.getValue(R: 1);
11418
11419 // Transform as necessary
11420 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11421 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11422 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11423 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11424 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11425 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11426
11427 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11428 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11429 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11430
11431 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11432}
11433
11434SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11435 SelectionDAG &DAG) const {
11436 EVT VT = Op.getValueType();
11437 Op = Op.getOperand(i: 0);
11438 EVT OpVT = Op.getValueType();
11439
11440 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11441
11442 SDLoc DL(Op);
11443
11444 // load a 0 vector for the third operand of VSUM.
11445 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11446
11447 // execute VSUM.
11448 switch (OpVT.getScalarSizeInBits()) {
11449 case 8:
11450 case 16:
11451 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11452 [[fallthrough]];
11453 case 32:
11454 case 64:
11455 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11456 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11457 break;
11458 case 128:
11459 break; // VSUM over v1i128 should not happen and would be a noop
11460 default:
11461 llvm_unreachable("Unexpected scalar size.");
11462 }
11463 // Cast to original vector type, retrieve last element.
11464 return DAG.getNode(
11465 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11466 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11467}
11468
11469static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
11470 FunctionType *FT = F->getFunctionType();
11471 const AttributeList &Attrs = F->getAttributes();
11472 if (Attrs.hasRetAttrs())
11473 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11474 OS << *F->getReturnType() << " @" << F->getName() << "(";
11475 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11476 if (I)
11477 OS << ", ";
11478 OS << *FT->getParamType(i: I);
11479 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11480 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11481 if (ArgAttrs.hasAttribute(Kind: A))
11482 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11483 }
11484 OS << ")\n";
11485}
11486
11487bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11488 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11489 if (Itr == IsInternalCache.end())
11490 Itr = IsInternalCache
11491 .insert(x: std::pair<const Function *, bool>(
11492 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11493 .first;
11494 return Itr->second;
11495}
11496
11497void SystemZTargetLowering::
11498verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
11499 const Function *F, SDValue Callee) const {
11500 // Temporarily only do the check when explicitly requested, until it can be
11501 // enabled by default.
11502 if (!EnableIntArgExtCheck)
11503 return;
11504
11505 bool IsInternal = false;
11506 const Function *CalleeFn = nullptr;
11507 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11508 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11509 IsInternal = isInternal(Fn: CalleeFn);
11510 if (!IsInternal && !verifyNarrowIntegerArgs(Outs)) {
11511 errs() << "ERROR: Missing extension attribute of passed "
11512 << "value in call to function:\n" << "Callee: ";
11513 if (CalleeFn != nullptr)
11514 printFunctionArgExts(F: CalleeFn, OS&: errs());
11515 else
11516 errs() << "-\n";
11517 errs() << "Caller: ";
11518 printFunctionArgExts(F, OS&: errs());
11519 llvm_unreachable("");
11520 }
11521}
11522
11523void SystemZTargetLowering::
11524verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
11525 const Function *F) const {
11526 // Temporarily only do the check when explicitly requested, until it can be
11527 // enabled by default.
11528 if (!EnableIntArgExtCheck)
11529 return;
11530
11531 if (!isInternal(Fn: F) && !verifyNarrowIntegerArgs(Outs)) {
11532 errs() << "ERROR: Missing extension attribute of returned "
11533 << "value from function:\n";
11534 printFunctionArgExts(F, OS&: errs());
11535 llvm_unreachable("");
11536 }
11537}
11538
11539// Verify that narrow integer arguments are extended as required by the ABI.
11540// Return false if an error is found.
11541bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11542 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
11543 if (!Subtarget.isTargetELF())
11544 return true;
11545
11546 if (EnableIntArgExtCheck.getNumOccurrences()) {
11547 if (!EnableIntArgExtCheck)
11548 return true;
11549 } else if (!getTargetMachine().Options.VerifyArgABICompliance)
11550 return true;
11551
11552 for (unsigned i = 0; i < Outs.size(); ++i) {
11553 MVT VT = Outs[i].VT;
11554 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11555 if (VT.isInteger()) {
11556 assert((VT == MVT::i32 || VT.getSizeInBits() >= 64) &&
11557 "Unexpected integer argument VT.");
11558 if (VT == MVT::i32 &&
11559 !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt())
11560 return false;
11561 }
11562 }
11563
11564 return true;
11565}
11566
11567void SystemZTargetLowering::insertSSPDeclarations(
11568 Module &M, const LibcallLoweringInfo &Libcalls) const {
11569 StringRef GuardMode = M.getStackProtectorGuard();
11570
11571 // In the TLS case, no symbol needs to be inserted.
11572 if (GuardMode == "tls" || GuardMode.empty())
11573 return;
11574
11575 // Otherwise (in the global case), insert the appropriate global variable.
11576 TargetLowering::insertSSPDeclarations(M, Libcalls);
11577}
11578