1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22#include "llvm/IR/GlobalAlias.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsS390.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
41static cl::opt<bool> EnableIntArgExtCheck(
42 "argext-abi-check", cl::init(Val: false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
87static MachineOperand earlyUseOperand(MachineOperand Op) {
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
93SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::i64;
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
105 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
109 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
110 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
113 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
114 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
126 addRegisterClass(VT: MVT::v8f16, RC: &SystemZ::VR128BitRegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
128 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
133 }
134
135 // Without vector support, model GR128 register pairs as v2i64.
136 if (!Subtarget.hasVector())
137 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::GR128BitRegClass);
138
139 // Compute derived properties from the register classes
140 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
141
142 // Set up special registers.
143 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
144
145 // TODO: It may be better to default to latency-oriented scheduling, however
146 // LLVM's current latency-oriented scheduler can't handle physreg definitions
147 // such as SystemZ has with CC, so set this to the register-pressure
148 // scheduler, because it can.
149 setSchedulingPreference(Sched::RegPressure);
150
151 setBooleanContents(ZeroOrOneBooleanContent);
152 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
153
154 setMaxAtomicSizeInBitsSupported(128);
155
156 // Instructions are strings of 2-byte aligned 2-byte values.
157 setMinFunctionAlignment(Align(2));
158 // For performance reasons we prefer 16-byte alignment.
159 setPrefFunctionAlignment(Align(16));
160
161 // Handle operations that are handled in a similar way for all types.
162 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
163 I <= MVT::LAST_FP_VALUETYPE;
164 ++I) {
165 MVT VT = MVT::SimpleValueType(I);
166 if (isTypeLegal(VT)) {
167 // Lower SET_CC into an IPM-based sequence.
168 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
169 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
170 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
171
172 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
173 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
174
175 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
176 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
177 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
178 }
179 }
180
181 // Expand jump table branches as address arithmetic followed by an
182 // indirect jump.
183 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
184
185 // Expand BRCOND into a BR_CC (see above).
186 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
187
188 // Handle integer types except i128.
189 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
190 I <= MVT::LAST_INTEGER_VALUETYPE;
191 ++I) {
192 MVT VT = MVT::SimpleValueType(I);
193 if (isTypeLegal(VT) && VT != MVT::i128) {
194 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
195
196 // Expand individual DIV and REMs into DIVREMs.
197 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
198 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
199 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
200 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
201 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
202 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
203
204 // Support addition/subtraction with overflow.
205 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
206 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
207
208 // Support addition/subtraction with carry.
209 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
210 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
211
212 // Support carry in as value rather than glue.
213 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
214 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
215
216 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
217 // available, or if the operand is constant.
218 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
219
220 // Use POPCNT on z196 and above.
221 if (Subtarget.hasPopulationCount())
222 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
223 else
224 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
225
226 // No special instructions for these.
227 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
228 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
229
230 // Use *MUL_LOHI where possible instead of MULH*.
231 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
232 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
233 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
234 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
235
236 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
237 // unsigned on z10 (only z196 and above have native support for
238 // unsigned conversions).
239 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
240 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
241 setOperationAction(Op, VT, Action: Custom);
242 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
243 setOperationAction(Op, VT, Action: Custom);
244 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
245 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
246 auto OpAction =
247 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
248 setOperationAction(Op, VT, Action: OpAction);
249 }
250 }
251 }
252
253 // Handle i128 if legal.
254 if (isTypeLegal(VT: MVT::i128)) {
255 // No special instructions for these.
256 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
257 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
258 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
259 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
260 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
261 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
262
263 // We may be able to use VSLDB/VSLD/VSRD for these.
264 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
265 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
266
267 // No special instructions for these before z17.
268 if (!Subtarget.hasVectorEnhancements3()) {
269 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
271 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
272 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
273 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
274 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
275 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
276 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
277 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
278 } else {
279 // Even if we do have a legal 128-bit multiply, we do not
280 // want 64-bit multiply-high operations to use it.
281 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
282 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
283 }
284
285 // Support addition/subtraction with carry.
286 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
287 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
288 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
289 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
290
291 // Use VPOPCT and add up partial results.
292 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
293
294 // Additional instructions available with z17.
295 if (Subtarget.hasVectorEnhancements3()) {
296 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
297
298 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
299 VT: MVT::i128, Action: Legal);
300 }
301 }
302
303 // These need custom handling in order to handle the f16 conversions.
304 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
305 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
306 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
307 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
308 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
309 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
310 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
311 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
312
313 // Lower 8- and 16-bit atomic operations into full i32 operations during
314 // type legalization.
315 setOperationAction(
316 Ops: {ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD, ISD::ATOMIC_LOAD_SUB,
317 ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR, ISD::ATOMIC_LOAD_XOR,
318 ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN, ISD::ATOMIC_LOAD_MAX,
319 ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
320 VTs: {MVT::i8, MVT::i16}, Action: Custom);
321
322 // Whether or not i128 is not a legal type, we need to custom lower
323 // the atomic operations in order to exploit SystemZ instructions.
324 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
325 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
326 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
327 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
328
329 // Mark sign/zero extending atomic loads as legal, which will make
330 // DAGCombiner fold extensions into atomic loads if possible.
331 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
332 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
333 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
334 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
335 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
336 MemVT: MVT::i8, Action: Legal);
337
338 // We can use the CC result of compare-and-swap to implement
339 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
340 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
341 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
342 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
343
344 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
345
346 // Traps are legal, as we will convert them to "j .+2".
347 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
348
349 // We have native support for a 64-bit CTLZ, via FLOGR.
350 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
351 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
352 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
353
354 // On z17 we have native support for a 64-bit CTTZ.
355 if (Subtarget.hasMiscellaneousExtensions4()) {
356 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
357 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
358 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
359 }
360
361 // On z15 we have native support for a 64-bit CTPOP.
362 if (Subtarget.hasMiscellaneousExtensions3()) {
363 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
364 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
365 }
366
367 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
368 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
369
370 // Expand 128 bit shifts without using a libcall.
371 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
372 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
373 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
374
375 // Also expand 256 bit shifts if i128 is a legal type.
376 if (isTypeLegal(VT: MVT::i128)) {
377 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
378 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
379 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
380 }
381
382 // Handle bitcast from fp128 to i128.
383 if (!isTypeLegal(VT: MVT::i128))
384 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
385
386 // We have native instructions for i8, i16 and i32 extensions, but not i1.
387 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
388 for (MVT VT : MVT::integer_valuetypes()) {
389 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
390 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
391 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
392 }
393
394 // Handle the various types of symbolic address.
395 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
396 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
397 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
398 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
399 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
400
401 // We need to handle dynamic allocations specially because of the
402 // 160-byte area at the bottom of the stack.
403 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
404 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
405
406 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
407 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
408
409 // Handle prefetches with PFD or PFDRL.
410 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
411
412 // Handle readcyclecounter with STCKF.
413 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
414
415 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
416 // Assume by default that all vector operations need to be expanded.
417 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
418 if (getOperationAction(Op: Opcode, VT) == Legal)
419 setOperationAction(Op: Opcode, VT, Action: Expand);
420
421 // Likewise all truncating stores and extending loads.
422 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
423 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
424 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
425 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
426 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
427 }
428
429 if (Subtarget.hasVector() && isTypeLegal(VT)) {
430 // These operations are legal for anything that can be stored in a
431 // vector register, even if there is no native support for the format
432 // as such. In particular, we can do these for v4f32 even though there
433 // are no specific instructions for that format.
434 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
435 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
436 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
437 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
438 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
439
440 // Likewise, except that we need to replace the nodes with something
441 // more specific.
442 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
443 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
444 }
445 }
446
447 if (!Subtarget.hasVector()) {
448 // A v2i64 GR128 pair can only be built from or split into its halves.
449 // Loads and stores are expanded to avoid tying up a register pair.
450 setOperationAction(Ops: {ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR, ISD::BITCAST,
451 ISD::UNDEF, ISD::POISON},
452 VT: MVT::v2i64, Action: Legal);
453 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
454 VT: MVT::v2i64, Action: Custom);
455 }
456
457 // Handle integer vector types.
458 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
459 if (Subtarget.hasVector() && isTypeLegal(VT)) {
460 // These operations have direct equivalents.
461 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
462 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
463 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
464 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
465 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
466 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
467 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
468 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
469 }
470 if (Subtarget.hasVectorEnhancements3() &&
471 VT != MVT::v16i8 && VT != MVT::v8i16) {
472 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
473 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
474 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
475 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
476 }
477 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
478 setOperationAction(Op: ISD::AND, VT, Action: Legal);
479 setOperationAction(Op: ISD::OR, VT, Action: Legal);
480 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
481 if (Subtarget.hasVectorEnhancements1())
482 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
483 else
484 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
485 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
486 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
487
488 // Convert a GPR scalar to a vector by inserting it into element 0.
489 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
490
491 // Use a series of unpacks for extensions.
492 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
493 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
494
495 // Detect shifts/rotates by a scalar amount and convert them into
496 // V*_BY_SCALAR.
497 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
498 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
499 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
500 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
501
502 // Add ISD::VECREDUCE_ADD as custom in order to implement
503 // it with VZERO+VSUM
504 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
505
506 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
507 // and inverting the result as necessary.
508 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
509
510 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT,
511 Action: Legal);
512 }
513 }
514
515 if (Subtarget.hasVector()) {
516 // There should be no need to check for float types other than v2f64
517 // since <2 x f32> isn't a legal type.
518 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
519 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
520 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
521 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
522 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
523 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
524 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
525 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
526
527 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
528 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
529 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
530 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
531 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
532 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
533 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
534 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
535 }
536
537 if (Subtarget.hasVectorEnhancements2()) {
538 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
539 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
540 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
541 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
542 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
543 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
544 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
545 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
546
547 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
548 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
549 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
550 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
551 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
552 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
553 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
554 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
555 }
556
557 // Handle floating-point types.
558 if (!useSoftFloat()) {
559 // Promote all f16 operations to float, with some exceptions below.
560 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
561 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
562 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
563 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
564 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
565 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
566 }
567 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
568 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
569 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
570 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
571 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
572
573 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
574 setOperationAction(Op, VT: MVT::f16, Action: Legal);
575 }
576
577 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
578 I <= MVT::LAST_FP_VALUETYPE;
579 ++I) {
580 MVT VT = MVT::SimpleValueType(I);
581 if (isTypeLegal(VT) && VT != MVT::f16) {
582 // We can use FI for FRINT.
583 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
584
585 // We can use the extended form of FI for other rounding operations.
586 if (Subtarget.hasFPExtension()) {
587 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
588 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
589 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
590 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
591 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
592 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
593 }
594
595 // No special instructions for these.
596 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
597 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
598 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
599 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
600 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
601
602 // Special treatment.
603 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
604
605 // Handle constrained floating-point operations.
606 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
607 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
608 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
609 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
610 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
611 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
612 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
613 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
614 if (Subtarget.hasFPExtension()) {
615 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
616 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
617 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
618 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
619 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
620 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
621 }
622
623 // Extension from f16 needs libcall.
624 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
625 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
626 }
627 }
628
629 // Handle floating-point vector types.
630 if (Subtarget.hasVector()) {
631 // Scalar-to-vector conversion is just a subreg.
632 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8f16, Action: Legal);
633 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
634 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
635
636 // Some insertions and extractions can be done directly but others
637 // need to go via integers.
638 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
639 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
640 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
641 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
642 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
643 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
644
645 // These operations have direct equivalents.
646 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
653 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
654 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
655 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
656 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
657 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
658 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
659 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
660 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
661
662 // Handle constrained floating-point operations.
663 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
664 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
665 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
666 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
667 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
668 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
669 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
670 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
671 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
672 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
673 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
674 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
675 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
676
677 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
678 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
679 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
680 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
681 if (Subtarget.hasVectorEnhancements1()) {
682 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
683 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
684 }
685 }
686
687 // The vector enhancements facility 1 has instructions for these.
688 if (Subtarget.hasVectorEnhancements1()) {
689 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
690 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
691 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
692 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
693 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
694 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
695 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
696 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
697 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
698 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
699 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
700 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
701 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
702 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
703 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
704
705 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
706 setOperationAction(Op: ISD::FMAXNUM, VT: Type, Action: Legal);
707 setOperationAction(Op: ISD::FMAXIMUM, VT: Type, Action: Legal);
708 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: Type, Action: Legal);
709 setOperationAction(Op: ISD::PSEUDO_FMAX, VT: Type, Action: Legal);
710 setOperationAction(Op: ISD::FMINNUM, VT: Type, Action: Legal);
711 setOperationAction(Op: ISD::FMINIMUM, VT: Type, Action: Legal);
712 setOperationAction(Op: ISD::FMINIMUMNUM, VT: Type, Action: Legal);
713 setOperationAction(Op: ISD::PSEUDO_FMIN, VT: Type, Action: Legal);
714 }
715
716 // Handle constrained floating-point operations.
717 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
720 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
721 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
722 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
723 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
724 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
725 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
726 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
728 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
729 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
730 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
731 MVT::v4f32, MVT::v2f64 }) {
732 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
733 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
734 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
735 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
736 setOperationAction(Op: ISD::STRICT_PSEUDO_FMAX, VT, Action: Legal);
737 setOperationAction(Op: ISD::STRICT_PSEUDO_FMIN, VT, Action: Legal);
738 }
739 }
740
741 // We only have fused f128 multiply-addition on vector registers.
742 if (!Subtarget.hasVectorEnhancements1()) {
743 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
744 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
745 }
746
747 // We don't have a copysign instruction on vector registers.
748 if (Subtarget.hasVectorEnhancements1())
749 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
750
751 // Needed so that we don't try to implement f128 constant loads using
752 // a load-and-extend of a f80 constant (in cases where the constant
753 // would fit in an f80).
754 for (MVT VT : MVT::fp_valuetypes())
755 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
756
757 // We don't have extending load instruction on vector registers.
758 if (Subtarget.hasVectorEnhancements1()) {
759 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
760 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
761 }
762
763 // Floating-point truncation and stores need to be done separately.
764 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
765 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
766 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
767
768 // We have 64-bit FPR<->GPR moves, but need special handling for
769 // 32-bit forms.
770 if (!Subtarget.hasVector()) {
771 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
772 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
773 }
774
775 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
776 // structure, but VAEND is a no-op.
777 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
778 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
779 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
780
781 if (Subtarget.isTargetzOS()) {
782 // Handle address space casts between mixed sized pointers.
783 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
784 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
785 }
786
787 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
788
789 // Codes for which we want to perform some z-specific combinations.
790 setTargetDAGCombine({ISD::ZERO_EXTEND,
791 ISD::SIGN_EXTEND,
792 ISD::SIGN_EXTEND_INREG,
793 ISD::LOAD,
794 ISD::STORE,
795 ISD::VECTOR_SHUFFLE,
796 ISD::EXTRACT_VECTOR_ELT,
797 ISD::FP_ROUND,
798 ISD::STRICT_FP_ROUND,
799 ISD::FP_EXTEND,
800 ISD::SINT_TO_FP,
801 ISD::UINT_TO_FP,
802 ISD::STRICT_FP_EXTEND,
803 ISD::FCOPYSIGN,
804 ISD::BSWAP,
805 ISD::SETCC,
806 ISD::SRL,
807 ISD::SRA,
808 ISD::MUL,
809 ISD::SDIV,
810 ISD::UDIV,
811 ISD::SREM,
812 ISD::UREM,
813 ISD::INTRINSIC_VOID,
814 ISD::INTRINSIC_W_CHAIN});
815
816 // Handle intrinsics.
817 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
818 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
819
820 // We're not using SJLJ for exception handling, but they're implemented
821 // solely to support use of __builtin_setjmp / __builtin_longjmp.
822 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
823 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
824
825 // We want to use MVC in preference to even a single load/store pair.
826 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
827 MaxStoresPerMemcpyOptSize = 0;
828
829 // Same with memmove.
830 MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
831 MaxStoresPerMemmoveOptSize = 0;
832
833 // The main memset sequence is a byte store followed by an MVC.
834 // Two STC or MV..I stores win over that, but the kind of fused stores
835 // generated by target-independent code don't when the byte value is
836 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
837 // than "STC;MVC". Handle the choice in target-specific code instead.
838 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
839 MaxStoresPerMemsetOptSize = 0;
840
841 // Default to having -disable-strictnode-mutation on
842 IsStrictFPEnabled = true;
843}
844
845bool SystemZTargetLowering::useSoftFloat() const {
846 return Subtarget.hasSoftFloat();
847}
848
849TargetLoweringBase::LegalizeTypeAction
850SystemZTargetLowering::getPreferredVectorAction(MVT VT) const {
851 // Widen subvectors to the full width rather than promoting integer
852 // elements. This is better because:
853 //
854 // (a) it means that we can handle the ABI for passing and returning
855 // sub-128 vectors without having to handle them as legal types.
856 //
857 // (b) we don't have instructions to extend on load and truncate on store,
858 // so promoting the integers is less efficient.
859 //
860 // (c) there are no multiplication instructions for the widest integer
861 // type (v2i64).
862
863 // Expand (narrow) f16 vectors during type legalization to avoid
864 // operations for all elements as with expansion after widening. Without
865 // vector support, v2i64 is only legal as a GR128 pair, so do not widen or
866 // promote to it.
867 if (!Subtarget.hasVector() || VT.getScalarType() == MVT::f16)
868 return VT.getVectorElementCount().isScalar() ? TypeScalarizeVector
869 : TypeSplitVector;
870 if (VT.getScalarSizeInBits() % 8 == 0)
871 return TypeWidenVector;
872 return TargetLoweringBase::getPreferredVectorAction(VT);
873}
874
875unsigned SystemZTargetLowering::getVectorTypeBreakdownForCallingConv(
876 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
877 unsigned &NumIntermediates, MVT &RegisterVT) const {
878 // Pass fp16 vectors in VR(s).
879 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16)) {
880 IntermediateVT = RegisterVT = MVT::v8f16;
881 return NumIntermediates =
882 divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
883 }
884 unsigned NumRegs = TargetLowering::getVectorTypeBreakdownForCallingConv(
885 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
886 // Without vector support, v2i64 is a GR128 pair, which is never used to pass
887 // values. Pass the elements individually.
888 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector()) {
889 IntermediateVT = RegisterVT = MVT::i64;
890 NumIntermediates *= 2;
891 NumRegs *= 2;
892 }
893 return NumRegs;
894}
895
896MVT SystemZTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
897 CallingConv::ID CC,
898 EVT VT) const {
899 // 128-bit single-element vector types are passed like other vectors,
900 // not like their element type.
901 if (Subtarget.hasVector() && VT.isVector() && VT.getSizeInBits() == 128 &&
902 VT.getVectorNumElements() == 1)
903 return MVT::v16i8;
904 // Pass fp16 vectors in VR(s).
905 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
906 return MVT::v8f16;
907 MVT RegisterVT =
908 TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
909 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
910 return MVT::i64;
911 return RegisterVT;
912}
913
914unsigned SystemZTargetLowering::getNumRegistersForCallingConv(
915 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
916 // Pass fp16 vectors in VR(s).
917 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
918 return divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
919 unsigned NumRegs =
920 TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
921 MVT RegisterVT =
922 TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
923 if (RegisterVT == MVT::v2i64 && !Subtarget.hasVector())
924 return NumRegs * 2;
925 return NumRegs;
926}
927
928EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
929 LLVMContext &, EVT VT) const {
930 if (!VT.isVector())
931 return MVT::i32;
932 return VT.changeVectorElementTypeToInteger();
933}
934
935bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
936 const MachineFunction &MF, EVT VT) const {
937 if (useSoftFloat())
938 return false;
939
940 VT = VT.getScalarType();
941
942 if (!VT.isSimple())
943 return false;
944
945 switch (VT.getSimpleVT().SimpleTy) {
946 case MVT::f32:
947 case MVT::f64:
948 return true;
949 case MVT::f128:
950 return Subtarget.hasVectorEnhancements1();
951 default:
952 break;
953 }
954
955 return false;
956}
957
958// Return true if the constant can be generated with a vector instruction,
959// such as VGM, VGMB or VREPI.
960bool SystemZVectorConstantInfo::isVectorConstantLegal(
961 const SystemZSubtarget &Subtarget) {
962 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
963 if (!Subtarget.hasVector() ||
964 (isFP128 && !Subtarget.hasVectorEnhancements1()))
965 return false;
966
967 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
968 // preferred way of creating all-zero and all-one vectors so give it
969 // priority over other methods below.
970 unsigned Mask = 0;
971 unsigned I = 0;
972 for (; I < SystemZ::VectorBytes; ++I) {
973 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
974 if (Byte == 0xff)
975 Mask |= 1ULL << I;
976 else if (Byte != 0)
977 break;
978 }
979 if (I == SystemZ::VectorBytes) {
980 Opcode = SystemZISD::BYTE_MASK;
981 OpVals.push_back(Elt: Mask);
982 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
983 return true;
984 }
985
986 if (SplatBitSize > 64)
987 return false;
988
989 auto TryValue = [&](uint64_t Value) -> bool {
990 // Try VECTOR REPLICATE IMMEDIATE
991 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
992 if (isInt<16>(x: SignedValue)) {
993 OpVals.push_back(Elt: ((unsigned) SignedValue));
994 Opcode = SystemZISD::REPLICATE;
995 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
996 NumElements: SystemZ::VectorBits / SplatBitSize);
997 return true;
998 }
999 // Try VECTOR GENERATE MASK
1000 unsigned Start, End;
1001 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
1002 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
1003 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
1004 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
1005 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
1006 OpVals.push_back(Elt: End - (64 - SplatBitSize));
1007 Opcode = SystemZISD::ROTATE_MASK;
1008 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
1009 NumElements: SystemZ::VectorBits / SplatBitSize);
1010 return true;
1011 }
1012 return false;
1013 };
1014
1015 // First try assuming that any undefined bits above the highest set bit
1016 // and below the lowest set bit are 1s. This increases the likelihood of
1017 // being able to use a sign-extended element value in VECTOR REPLICATE
1018 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
1019 uint64_t SplatBitsZ = SplatBits.getZExtValue();
1020 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
1021 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
1022 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
1023 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
1024 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
1025 if (TryValue(SplatBitsZ | Upper | Lower))
1026 return true;
1027
1028 // Now try assuming that any undefined bits between the first and
1029 // last defined set bits are set. This increases the chances of
1030 // using a non-wraparound mask.
1031 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
1032 return TryValue(SplatBitsZ | Middle);
1033}
1034
1035SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
1036 if (IntImm.isSingleWord()) {
1037 IntBits = APInt(128, IntImm.getZExtValue());
1038 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
1039 } else
1040 IntBits = IntImm;
1041 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
1042
1043 // Find the smallest splat.
1044 SplatBits = IntImm;
1045 unsigned Width = SplatBits.getBitWidth();
1046 while (Width > 8) {
1047 unsigned HalfSize = Width / 2;
1048 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
1049 APInt LowValue = SplatBits.trunc(width: HalfSize);
1050
1051 // If the two halves do not match, stop here.
1052 if (HighValue != LowValue || 8 > HalfSize)
1053 break;
1054
1055 SplatBits = HighValue;
1056 Width = HalfSize;
1057 }
1058 SplatUndef = 0;
1059 SplatBitSize = Width;
1060}
1061
1062SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
1063 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1064 bool HasAnyUndefs;
1065
1066 // Get IntBits by finding the 128 bit splat.
1067 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
1068 isBigEndian: true);
1069
1070 // Get SplatBits by finding the 8 bit or greater splat.
1071 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
1072 isBigEndian: true);
1073}
1074
1075TargetLowering::FPImmAction
1076SystemZTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
1077 bool ForCodeSize) const {
1078 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1079 if (Imm.isZero() || Imm.isNegZero())
1080 return FPImmAction::Legal;
1081
1082 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget)
1083 ? FPImmAction::Legal
1084 : FPImmAction::ConstantPool;
1085}
1086
1087MachineBasicBlock *
1088SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
1089 MachineBasicBlock *MBB) const {
1090 DebugLoc DL = MI.getDebugLoc();
1091 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1092 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1093
1094 MachineFunction *MF = MBB->getParent();
1095 MachineRegisterInfo &MRI = MF->getRegInfo();
1096
1097 const BasicBlock *BB = MBB->getBasicBlock();
1098 MachineFunction::iterator I = ++MBB->getIterator();
1099
1100 Register DstReg = MI.getOperand(i: 0).getReg();
1101 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1102 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1103 (void)TRI;
1104 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1105 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1106
1107 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1108 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1109 // For v = setjmp(buf), we generate.
1110 // Algorithm:
1111 //
1112 // ---------
1113 // | thisMBB |
1114 // ---------
1115 // |
1116 // ------------------------
1117 // | |
1118 // ---------- ---------------
1119 // | mainMBB | | restoreMBB |
1120 // | v = 0 | | v = 1 |
1121 // ---------- ---------------
1122 // | |
1123 // -------------------------
1124 // |
1125 // -----------------------------
1126 // | sinkMBB |
1127 // | phi(v_mainMBB,v_restoreMBB) |
1128 // -----------------------------
1129 // thisMBB:
1130 // buf[FPOffset] = Frame Pointer if hasFP.
1131 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1132 // buf[BCOffset] = Backchain value if building with -mbackchain.
1133 // buf[SPOffset] = Stack Pointer.
1134 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1135 // SjLjSetup restoreMBB
1136 // mainMBB:
1137 // v_main = 0
1138 // sinkMBB:
1139 // v = phi(v_main, v_restore)
1140 // restoreMBB:
1141 // v_restore = 1
1142
1143 MachineBasicBlock *ThisMBB = MBB;
1144 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1145 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1146 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1147
1148 MF->insert(MBBI: I, MBB: MainMBB);
1149 MF->insert(MBBI: I, MBB: SinkMBB);
1150 MF->push_back(MBB: RestoreMBB);
1151 RestoreMBB->setMachineBlockAddressTaken();
1152
1153 MachineInstrBuilder MIB;
1154
1155 // Transfer the remainder of BB and its successor edges to sinkMBB.
1156 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1157 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1158 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1159
1160 // thisMBB:
1161 const int64_t FPOffset = 0; // Slot 1.
1162 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1163 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1164 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1165
1166 // Buf address.
1167 Register BufReg = MI.getOperand(i: 1).getReg();
1168
1169 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1170 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1171
1172 // Prepare IP for longjmp.
1173 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1174 .addMBB(MBB: RestoreMBB);
1175 // Store IP for return from jmp, slot 2, offset = 1.
1176 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1177 .addReg(RegNo: LabelReg)
1178 .addReg(RegNo: BufReg)
1179 .addImm(Val: LabelOffset)
1180 .addReg(RegNo: 0);
1181
1182 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1183 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1184 if (HasFP) {
1185 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1186 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1187 .addReg(RegNo: BufReg)
1188 .addImm(Val: FPOffset)
1189 .addReg(RegNo: 0);
1190 }
1191
1192 // Store SP.
1193 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1194 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1195 .addReg(RegNo: BufReg)
1196 .addImm(Val: SPOffset)
1197 .addReg(RegNo: 0);
1198
1199 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1200 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1201 if (BackChain) {
1202 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1203 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1204 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1205 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1206 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1207 .addReg(RegNo: 0);
1208
1209 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1210 .addReg(RegNo: BCReg)
1211 .addReg(RegNo: BufReg)
1212 .addImm(Val: BCOffset)
1213 .addReg(RegNo: 0);
1214 }
1215
1216 // Setup.
1217 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1218 .addMBB(MBB: RestoreMBB);
1219
1220 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1221 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1222
1223 ThisMBB->addSuccessor(Succ: MainMBB);
1224 ThisMBB->addSuccessor(Succ: RestoreMBB);
1225
1226 // mainMBB:
1227 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1228 MainMBB->addSuccessor(Succ: SinkMBB);
1229
1230 // sinkMBB:
1231 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1232 .addReg(RegNo: MainDstReg)
1233 .addMBB(MBB: MainMBB)
1234 .addReg(RegNo: RestoreDstReg)
1235 .addMBB(MBB: RestoreMBB);
1236
1237 // restoreMBB.
1238 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1239 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1240 RestoreMBB->addSuccessor(Succ: SinkMBB);
1241
1242 MI.eraseFromParent();
1243
1244 return SinkMBB;
1245}
1246
1247MachineBasicBlock *
1248SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1249 MachineBasicBlock *MBB) const {
1250
1251 DebugLoc DL = MI.getDebugLoc();
1252 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1253
1254 MachineFunction *MF = MBB->getParent();
1255 MachineRegisterInfo &MRI = MF->getRegInfo();
1256
1257 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1258 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1259 Register BufReg = MI.getOperand(i: 0).getReg();
1260 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1261 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1262
1263 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1264 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1265
1266 MachineInstrBuilder MIB;
1267
1268 const int64_t FPOffset = 0;
1269 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1270 const int64_t BCOffset = 2 * PVT.getStoreSize();
1271 const int64_t SPOffset = 3 * PVT.getStoreSize();
1272 const int64_t LPOffset = 4 * PVT.getStoreSize();
1273
1274 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1275 .addReg(RegNo: BufReg)
1276 .addImm(Val: LabelOffset)
1277 .addReg(RegNo: 0);
1278
1279 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1280 DestReg: SpecialRegs->getFramePointerRegister())
1281 .addReg(RegNo: BufReg)
1282 .addImm(Val: FPOffset)
1283 .addReg(RegNo: 0);
1284
1285 // We are restoring R13 even though we never stored in setjmp from llvm,
1286 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1287 // gcc setjmp and llvm longjmp.
1288 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1289 .addReg(RegNo: BufReg)
1290 .addImm(Val: LPOffset)
1291 .addReg(RegNo: 0);
1292
1293 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1294 if (BackChain) {
1295 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1296 .addReg(RegNo: BufReg)
1297 .addImm(Val: BCOffset)
1298 .addReg(RegNo: 0);
1299 }
1300
1301 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1302 DestReg: SpecialRegs->getStackPointerRegister())
1303 .addReg(RegNo: BufReg)
1304 .addImm(Val: SPOffset)
1305 .addReg(RegNo: 0);
1306
1307 if (BackChain) {
1308 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1309 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1310 .addReg(RegNo: BCReg)
1311 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1312 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1313 .addReg(RegNo: 0);
1314 }
1315
1316 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1317
1318 MI.eraseFromParent();
1319 return MBB;
1320}
1321
1322bool SystemZTargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1323 const MachineFunction &MF) const {
1324 // Without vector support, a v2i64 store would just be split again.
1325 return MemVT != MVT::v2i64 || Subtarget.hasVector();
1326}
1327
1328bool SystemZTargetLowering::aggressivelyPreferBuildVectorSources(
1329 EVT VecVT) const {
1330 // Without vector support, a v2i64 is just a pair of GPRs.
1331 return !Subtarget.hasVector();
1332}
1333
1334/// Returns true if stack probing through inline assembly is requested.
1335bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1336 // If the function specifically requests inline stack probes, emit them.
1337 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1338 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1339 "inline-asm";
1340 return false;
1341}
1342
1343TargetLowering::AtomicExpansionKind
1344SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1345 return AtomicExpansionKind::None;
1346}
1347
1348TargetLowering::AtomicExpansionKind
1349SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1350 return AtomicExpansionKind::None;
1351}
1352
1353TargetLowering::AtomicExpansionKind
1354SystemZTargetLowering::shouldExpandAtomicRMWInIR(
1355 const AtomicRMWInst *RMW) const {
1356 // Don't expand subword operations as they require special treatment.
1357 if (RMW->getType()->isIntegerTy(BitWidth: 8) || RMW->getType()->isIntegerTy(BitWidth: 16))
1358 return AtomicExpansionKind::None;
1359
1360 // Don't expand if there is a target instruction available.
1361 if (Subtarget.hasInterlockedAccess1() &&
1362 (RMW->getType()->isIntegerTy(BitWidth: 32) || RMW->getType()->isIntegerTy(BitWidth: 64)) &&
1363 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1364 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1365 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1366 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1367 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1368 return AtomicExpansionKind::None;
1369
1370 return AtomicExpansionKind::CmpXChg;
1371}
1372
1373bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1374 // We can use CGFI or CLGFI.
1375 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1376}
1377
1378bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1379 // We can use ALGFI or SLGFI.
1380 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1381}
1382
1383bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1384 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1385 // Unaligned accesses should never be slower than the expanded version.
1386 // We check specifically for aligned accesses in the few cases where
1387 // they are required.
1388 if (Fast)
1389 *Fast = 1;
1390 return true;
1391}
1392
1393bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1394 EVT VT = Y.getValueType();
1395
1396 // We can use NC(G)RK for types in GPRs ...
1397 if (VT == MVT::i32 || VT == MVT::i64)
1398 return Subtarget.hasMiscellaneousExtensions3();
1399
1400 // ... or VNC for types in VRs.
1401 if (VT.isVector() || VT == MVT::i128)
1402 return Subtarget.hasVector();
1403
1404 return false;
1405}
1406
1407// Information about the addressing mode for a memory access.
1408struct AddressingMode {
1409 // True if a long displacement is supported.
1410 bool LongDisplacement;
1411
1412 // True if use of index register is supported.
1413 bool IndexReg;
1414
1415 AddressingMode(bool LongDispl, bool IdxReg) :
1416 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1417};
1418
1419// Return the desired addressing mode for a Load which has only one use (in
1420// the same block) which is a Store.
1421static AddressingMode getLoadStoreAddrMode(bool HasVector,
1422 Type *Ty) {
1423 // With vector support a Load->Store combination may be combined to either
1424 // an MVC or vector operations and it seems to work best to allow the
1425 // vector addressing mode.
1426 if (HasVector)
1427 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1428
1429 // Otherwise only the MVC case is special.
1430 bool MVC = Ty->isIntegerTy(BitWidth: 8);
1431 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1432}
1433
1434// Return the addressing mode which seems most desirable given an LLVM
1435// Instruction pointer.
1436static AddressingMode
1437supportedAddressingMode(Instruction *I, bool HasVector) {
1438 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1439 switch (II->getIntrinsicID()) {
1440 default: break;
1441 case Intrinsic::memset:
1442 case Intrinsic::memmove:
1443 case Intrinsic::memcpy:
1444 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1445 }
1446 }
1447
1448 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1449 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1450 if (SingleUser->getParent() == I->getParent()) {
1451 if (isa<ICmpInst>(Val: SingleUser)) {
1452 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1453 if (C->getBitWidth() <= 64 &&
1454 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1455 // Comparison of memory with 16 bit signed / unsigned immediate
1456 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1457 } else if (isa<StoreInst>(Val: SingleUser))
1458 // Load->Store
1459 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1460 }
1461 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1462 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1463 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1464 // Load->Store
1465 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1466 }
1467
1468 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1469
1470 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1471 // dependencies (LDE only supports small offsets).
1472 // * Utilize the vector registers to hold floating point
1473 // values (vector load / store instructions only support small
1474 // offsets).
1475
1476 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1477 I->getOperand(i: 0)->getType());
1478 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1479 bool IsVectorAccess = MemAccessTy->isVectorTy();
1480
1481 // A store of an extracted vector element will be combined into a VSTE type
1482 // instruction.
1483 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1484 Value *DataOp = I->getOperand(i: 0);
1485 if (isa<ExtractElementInst>(Val: DataOp))
1486 IsVectorAccess = true;
1487 }
1488
1489 // A load which gets inserted into a vector element will be combined into a
1490 // VLE type instruction.
1491 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1492 User *LoadUser = *I->user_begin();
1493 if (isa<InsertElementInst>(Val: LoadUser))
1494 IsVectorAccess = true;
1495 }
1496
1497 if (IsFPAccess || IsVectorAccess)
1498 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1499 }
1500
1501 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1502}
1503
1504bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1505 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1506 // Punt on globals for now, although they can be used in limited
1507 // RELATIVE LONG cases.
1508 if (AM.BaseGV)
1509 return false;
1510
1511 // Require a 20-bit signed offset.
1512 if (!isInt<20>(x: AM.BaseOffs))
1513 return false;
1514
1515 bool RequireD12 =
1516 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(BitWidth: 128));
1517 AddressingMode SupportedAM(!RequireD12, true);
1518 if (I != nullptr)
1519 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1520
1521 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1522 return false;
1523
1524 if (!SupportedAM.IndexReg)
1525 // No indexing allowed.
1526 return AM.Scale == 0;
1527 else
1528 // Indexing is OK but no scale factor can be applied.
1529 return AM.Scale == 0 || AM.Scale == 1;
1530}
1531
1532bool SystemZTargetLowering::findOptimalMemOpLowering(
1533 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1534 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1535 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1536
1537 assert(Limit != ~0U &&
1538 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1539
1540 if (Op.isZeroMemset())
1541 return false; // Memset zero: Use XC.
1542
1543 const int MVCFastLen = 16;
1544 // Use MVC up to 16 bytes for memcpy. Small memset uses STC/MVI for first
1545 // byte.
1546 if (Op.isMemcpy() && Op.size() <= MVCFastLen)
1547 return false;
1548 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1549 return false;
1550
1551 // Avoid unaligned VL/VST:s.
1552 if ((Op.size() >= 16 && !Op.isAligned(AlignCheck: Align(8))) ||
1553 (Op.size() >= 25 && Op.size() <= 31))
1554 return false;
1555
1556 return TargetLowering::findOptimalMemOpLowering(
1557 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1558}
1559
1560EVT SystemZTargetLowering::getOptimalMemOpType(
1561 LLVMContext &Context, const MemOp &Op,
1562 const AttributeList &FuncAttributes) const {
1563 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1564}
1565
1566bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1567 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1568 return false;
1569 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1570 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1571 return FromBits > ToBits;
1572}
1573
1574bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1575 if (!FromVT.isInteger() || !ToVT.isInteger())
1576 return false;
1577 unsigned FromBits = FromVT.getFixedSizeInBits();
1578 unsigned ToBits = ToVT.getFixedSizeInBits();
1579 return FromBits > ToBits;
1580}
1581
1582bool SystemZTargetLowering::isTypeDesirableForOp(SDNode *N, EVT VT) const {
1583 unsigned Opc = N->getOpcode();
1584 if ((Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP) && VT.isVector() &&
1585 VT.getScalarSizeInBits() < N->getValueType(ResNo: 0).getScalarSizeInBits())
1586 return false;
1587 return TargetLowering::isTypeDesirableForOp(N, VT);
1588}
1589
1590//===----------------------------------------------------------------------===//
1591// Inline asm support
1592//===----------------------------------------------------------------------===//
1593
1594TargetLowering::ConstraintType
1595SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1596 if (Constraint.size() == 1) {
1597 switch (Constraint[0]) {
1598 case 'a': // Address register
1599 case 'd': // Data register (equivalent to 'r')
1600 case 'f': // Floating-point register
1601 case 'h': // High-part register
1602 case 'r': // General-purpose register
1603 case 'v': // Vector register
1604 return C_RegisterClass;
1605
1606 case 'Q': // Memory with base and unsigned 12-bit displacement
1607 case 'R': // Likewise, plus an index
1608 case 'S': // Memory with base and signed 20-bit displacement
1609 case 'T': // Likewise, plus an index
1610 case 'm': // Equivalent to 'T'.
1611 return C_Memory;
1612
1613 case 'I': // Unsigned 8-bit constant
1614 case 'J': // Unsigned 12-bit constant
1615 case 'K': // Signed 16-bit constant
1616 case 'L': // Signed 20-bit displacement (on all targets we support)
1617 case 'M': // 0x7fffffff
1618 return C_Immediate;
1619
1620 default:
1621 break;
1622 }
1623 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1624 switch (Constraint[1]) {
1625 case 'Q': // Address with base and unsigned 12-bit displacement
1626 case 'R': // Likewise, plus an index
1627 case 'S': // Address with base and signed 20-bit displacement
1628 case 'T': // Likewise, plus an index
1629 return C_Address;
1630
1631 default:
1632 break;
1633 }
1634 } else if (Constraint.size() == 5 && Constraint.starts_with(Prefix: "{")) {
1635 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1636 return C_Other;
1637 }
1638 return TargetLowering::getConstraintType(Constraint);
1639}
1640
1641TargetLowering::ConstraintWeight
1642SystemZTargetLowering::getSingleConstraintMatchWeight(
1643 AsmOperandInfo &Info, const char *Constraint) const {
1644 ConstraintWeight Weight = CW_Invalid;
1645 Value *CallOperandVal = Info.CallOperandVal;
1646 // If we don't have a value, we can't do a match,
1647 // but allow it at the lowest weight.
1648 if (!CallOperandVal)
1649 return CW_Default;
1650 Type *type = CallOperandVal->getType();
1651 // Look at the constraint type.
1652 switch (*Constraint) {
1653 default:
1654 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1655 break;
1656
1657 case 'a': // Address register
1658 case 'd': // Data register (equivalent to 'r')
1659 case 'h': // High-part register
1660 case 'r': // General-purpose register
1661 Weight =
1662 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1663 break;
1664
1665 case 'f': // Floating-point register
1666 if (!useSoftFloat())
1667 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1668 break;
1669
1670 case 'v': // Vector register
1671 if (Subtarget.hasVector())
1672 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1673 : CW_Default;
1674 break;
1675
1676 case 'I': // Unsigned 8-bit constant
1677 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1678 if (isUInt<8>(x: C->getZExtValue()))
1679 Weight = CW_Constant;
1680 break;
1681
1682 case 'J': // Unsigned 12-bit constant
1683 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1684 if (isUInt<12>(x: C->getZExtValue()))
1685 Weight = CW_Constant;
1686 break;
1687
1688 case 'K': // Signed 16-bit constant
1689 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1690 if (isInt<16>(x: C->getSExtValue()))
1691 Weight = CW_Constant;
1692 break;
1693
1694 case 'L': // Signed 20-bit displacement (on all targets we support)
1695 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1696 if (isInt<20>(x: C->getSExtValue()))
1697 Weight = CW_Constant;
1698 break;
1699
1700 case 'M': // 0x7fffffff
1701 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1702 if (C->getZExtValue() == 0x7fffffff)
1703 Weight = CW_Constant;
1704 break;
1705 }
1706 return Weight;
1707}
1708
1709// Parse a "{tNNN}" register constraint for which the register type "t"
1710// has already been verified. MC is the class associated with "t" and
1711// Map maps 0-based register numbers to LLVM register numbers.
1712static std::pair<unsigned, const TargetRegisterClass *>
1713parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1714 const unsigned *Map, unsigned Size) {
1715 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1716 if (isdigit(Constraint[2])) {
1717 unsigned Index;
1718 bool Failed =
1719 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1720 if (!Failed && Index < Size && Map[Index])
1721 return std::make_pair(x: Map[Index], y&: RC);
1722 }
1723 return std::make_pair(x: 0U, y: nullptr);
1724}
1725
1726std::pair<unsigned, const TargetRegisterClass *>
1727SystemZTargetLowering::getRegForInlineAsmConstraint(
1728 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1729 if (Constraint.size() == 1) {
1730 // GCC Constraint Letters
1731 switch (Constraint[0]) {
1732 default: break;
1733 case 'd': // Data register (equivalent to 'r')
1734 case 'r': // General-purpose register
1735 // MVT::Other has no size, e.g. an aggregate operand that could not be
1736 // converted to an integer type.
1737 if (VT == MVT::Other)
1738 break;
1739 if (VT.getSizeInBits() == 64)
1740 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1741 else if (VT.getSizeInBits() == 128)
1742 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1743 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1744
1745 case 'a': // Address register
1746 if (VT == MVT::i64)
1747 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1748 else if (VT == MVT::i128)
1749 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1750 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1751
1752 case 'h': // High-part register (an LLVM extension)
1753 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1754
1755 case 'f': // Floating-point register
1756 if (!useSoftFloat()) {
1757 if (VT == MVT::Other)
1758 break;
1759 if (VT.getSizeInBits() == 16)
1760 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1761 else if (VT.getSizeInBits() == 64)
1762 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1763 else if (VT.getSizeInBits() == 128)
1764 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1765 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1766 }
1767 break;
1768
1769 case 'v': // Vector register
1770 if (Subtarget.hasVector()) {
1771 if (VT == MVT::Other)
1772 break;
1773 if (VT.getSizeInBits() == 16)
1774 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1775 if (VT.getSizeInBits() == 32)
1776 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1777 if (VT.getSizeInBits() == 64)
1778 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1779 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1780 }
1781 break;
1782 }
1783 }
1784 if (Constraint.starts_with(Prefix: "{")) {
1785
1786 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1787 // to check the size on.
1788 auto getVTSizeInBits = [&VT]() {
1789 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1790 };
1791
1792 // We need to override the default register parsing for GPRs and FPRs
1793 // because the interpretation depends on VT. The internal names of
1794 // the registers are also different from the external names
1795 // (F0D and F0S instead of F0, etc.).
1796 if (Constraint[1] == 'r') {
1797 if (getVTSizeInBits() == 32)
1798 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1799 Map: SystemZMC::GR32Regs, Size: 16);
1800 if (getVTSizeInBits() == 128)
1801 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1802 Map: SystemZMC::GR128Regs, Size: 16);
1803 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1804 Map: SystemZMC::GR64Regs, Size: 16);
1805 }
1806 if (Constraint[1] == 'f') {
1807 if (useSoftFloat())
1808 return std::make_pair(
1809 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1810 if (getVTSizeInBits() == 16)
1811 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1812 Map: SystemZMC::FP16Regs, Size: 16);
1813 if (getVTSizeInBits() == 32)
1814 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1815 Map: SystemZMC::FP32Regs, Size: 16);
1816 if (getVTSizeInBits() == 128)
1817 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1818 Map: SystemZMC::FP128Regs, Size: 16);
1819 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1820 Map: SystemZMC::FP64Regs, Size: 16);
1821 }
1822 if (Constraint[1] == 'v') {
1823 if (!Subtarget.hasVector())
1824 return std::make_pair(
1825 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1826 if (getVTSizeInBits() == 16)
1827 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1828 Map: SystemZMC::VR16Regs, Size: 32);
1829 if (getVTSizeInBits() == 32)
1830 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1831 Map: SystemZMC::VR32Regs, Size: 32);
1832 if (getVTSizeInBits() == 64)
1833 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1834 Map: SystemZMC::VR64Regs, Size: 32);
1835 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1836 Map: SystemZMC::VR128Regs, Size: 32);
1837 }
1838 if (Constraint[1] == '@') {
1839 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1840 return std::make_pair(x: SystemZ::CC, y: &SystemZ::CCRRegClass);
1841 }
1842 }
1843 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1844}
1845
1846// FIXME? Maybe this could be a TableGen attribute on some registers and
1847// this table could be generated automatically from RegInfo.
1848Register
1849SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1850 const MachineFunction &MF) const {
1851 Register Reg =
1852 StringSwitch<Register>(RegName)
1853 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1854 : SystemZ::NoRegister)
1855 .Case(S: "r15",
1856 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1857 .Default(Value: Register());
1858
1859 return Reg;
1860}
1861
1862Register SystemZTargetLowering::getExceptionPointerRegister(
1863 ExceptionHandling EH, const Constant *PersonalityFn) const {
1864 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1865}
1866
1867Register SystemZTargetLowering::getExceptionSelectorRegister(
1868 ExceptionHandling EH, const Constant *PersonalityFn) const {
1869 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1870}
1871
1872// Convert condition code in CCReg to an i32 value.
1873static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
1874 SDLoc DL(CCReg);
1875 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
1876 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
1877 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
1878}
1879
1880// Lower @cc targets via setcc.
1881SDValue SystemZTargetLowering::LowerAsmOutputForConstraint(
1882 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1883 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1884 if (StringRef("{@cc}").compare(RHS: OpInfo.ConstraintCode) != 0)
1885 return SDValue();
1886
1887 // Check that return type is valid.
1888 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1889 OpInfo.ConstraintVT.getSizeInBits() < 8)
1890 report_fatal_error(reason: "Glue output operand is of invalid type");
1891
1892 if (Glue.getNode()) {
1893 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32, Glue);
1894 Chain = Glue.getValue(R: 1);
1895 } else
1896 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32);
1897 return getCCResult(DAG, CCReg: Glue);
1898}
1899
1900void SystemZTargetLowering::LowerAsmOperandForConstraint(
1901 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1902 SelectionDAG &DAG) const {
1903 // Only support length 1 constraints for now.
1904 if (Constraint.size() == 1) {
1905 switch (Constraint[0]) {
1906 case 'I': // Unsigned 8-bit constant
1907 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1908 if (isUInt<8>(x: C->getZExtValue()))
1909 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1910 VT: Op.getValueType()));
1911 return;
1912
1913 case 'J': // Unsigned 12-bit constant
1914 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1915 if (isUInt<12>(x: C->getZExtValue()))
1916 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1917 VT: Op.getValueType()));
1918 return;
1919
1920 case 'K': // Signed 16-bit constant
1921 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1922 if (isInt<16>(x: C->getSExtValue()))
1923 Ops.push_back(x: DAG.getSignedTargetConstant(
1924 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1925 return;
1926
1927 case 'L': // Signed 20-bit displacement (on all targets we support)
1928 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1929 if (isInt<20>(x: C->getSExtValue()))
1930 Ops.push_back(x: DAG.getSignedTargetConstant(
1931 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1932 return;
1933
1934 case 'M': // 0x7fffffff
1935 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1936 if (C->getZExtValue() == 0x7fffffff)
1937 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1938 VT: Op.getValueType()));
1939 return;
1940 }
1941 }
1942 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1943}
1944
1945//===----------------------------------------------------------------------===//
1946// Calling conventions
1947//===----------------------------------------------------------------------===//
1948
1949#define GET_CALLING_CONV_IMPL
1950#include "SystemZGenCallingConv.inc"
1951
1952const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1953 CallingConv::ID) const {
1954 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1955 SystemZ::R14D, 0 };
1956 return ScratchRegs;
1957}
1958
1959bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1960 Type *ToType) const {
1961 return isTruncateFree(FromType, ToType);
1962}
1963
1964bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1965 return CI->isTailCall();
1966}
1967
1968// Value is a value that has been passed to us in the location described by VA
1969// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1970// any loads onto Chain.
1971static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1972 CCValAssign &VA, SDValue Chain,
1973 SDValue Value) {
1974 // If the argument has been promoted from a smaller type, insert an
1975 // assertion to capture this.
1976 if (VA.getLocInfo() == CCValAssign::SExt)
1977 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1978 N2: DAG.getValueType(VA.getValVT()));
1979 else if (VA.getLocInfo() == CCValAssign::ZExt)
1980 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1981 N2: DAG.getValueType(VA.getValVT()));
1982
1983 if (VA.isExtInLoc())
1984 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1985 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1986 // If this is a short vector argument loaded from the stack,
1987 // extend from i64 to full vector size and then bitcast.
1988 assert(VA.getLocVT() == MVT::i64);
1989 assert(VA.getValVT().isVector());
1990 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1991 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1992 } else
1993 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1994 return Value;
1995}
1996
1997// Value is a value of type VA.getValVT() that we need to copy into
1998// the location described by VA. Return a copy of Value converted to
1999// VA.getValVT(). The caller is responsible for handling indirect values.
2000static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
2001 CCValAssign &VA, SDValue Value) {
2002 switch (VA.getLocInfo()) {
2003 case CCValAssign::SExt:
2004 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
2005 case CCValAssign::ZExt:
2006 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
2007 case CCValAssign::AExt:
2008 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
2009 case CCValAssign::BCvt: {
2010 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
2011 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
2012 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
2013 // For an f32 vararg we need to first promote it to an f64 and then
2014 // bitcast it to an i64.
2015 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
2016 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
2017 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
2018 ? MVT::v2i64
2019 : VA.getLocVT();
2020 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
2021 // For ELF, this is a short vector argument to be stored to the stack,
2022 // bitcast to v2i64 and then extract first element.
2023 if (BitCastToType == MVT::v2i64)
2024 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
2025 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
2026 return Value;
2027 }
2028 case CCValAssign::Full:
2029 return Value;
2030 default:
2031 llvm_unreachable("Unhandled getLocInfo()");
2032 }
2033}
2034
2035static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
2036 SDLoc DL(In);
2037 SDValue Lo, Hi;
2038 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
2039 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
2040 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
2041 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
2042 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
2043 } else {
2044 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
2045 }
2046
2047 return DAG.getNode(Opcode: SystemZISD::PAIR128, DL, VT: MVT::v2i64, N1: Hi, N2: Lo);
2048}
2049
2050static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
2051 SDLoc DL(In);
2052 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
2053 DL, VT: MVT::i64, Operand: In);
2054 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
2055 DL, VT: MVT::i64, Operand: In);
2056
2057 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
2058 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
2059 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
2060 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
2061 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
2062 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
2063 } else {
2064 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
2065 }
2066}
2067
2068bool SystemZTargetLowering::splitValueIntoRegisterParts(
2069 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
2070 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
2071 EVT ValueVT = Val.getValueType();
2072 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2073 // Inline assembly operand.
2074 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
2075 return true;
2076 }
2077
2078 return false;
2079}
2080
2081SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
2082 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
2083 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
2084 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2085 // Inline assembly operand.
2086 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
2087 return DAG.getBitcast(VT: ValueVT, V: Res);
2088 }
2089
2090 return SDValue();
2091}
2092
2093// The first part of a split stack argument is at index I in Args (and
2094// ArgLocs). Return the type of a part and the number of them by reference.
2095template <class ArgTy>
2096static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args,
2097 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2098 MVT &PartVT, unsigned &NumParts) {
2099 if (!Args[I].Flags.isSplit())
2100 return false;
2101 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2102 "ArgLocs havoc.");
2103 PartVT = ArgLocs[I].getValVT();
2104 NumParts = 1;
2105 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2106 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2107 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2108 ++NumParts;
2109 if (Args[PartIdx].Flags.isSplitEnd())
2110 break;
2111 }
2112 return true;
2113}
2114
2115// Return the virtual register holding the incoming "ADA" special register (R5)
2116// of an XPLINK function, creating it on first use. LowerFormalArguments is not
2117// called for naked functions, but those still get R5 on entry and may use it to
2118// address globals and functions.
2119static Register getOrCreateADAVirtualRegister(MachineFunction &MF) {
2120 SystemZMachineFunctionInfo *FuncInfo =
2121 MF.getInfo<SystemZMachineFunctionInfo>();
2122 Register ADAvReg = FuncInfo->getADAVirtualRegister();
2123 if (ADAvReg)
2124 return ADAvReg;
2125
2126 MachineRegisterInfo &MRI = MF.getRegInfo();
2127 ADAvReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
2128 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2129 MF.getSubtarget<SystemZSubtarget>().getSpecialRegisters());
2130 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2131 FuncInfo->setADAVirtualRegister(ADAvReg);
2132 return ADAvReg;
2133}
2134
2135SDValue SystemZTargetLowering::LowerFormalArguments(
2136 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2137 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2138 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2139 MachineFunction &MF = DAG.getMachineFunction();
2140 MachineFrameInfo &MFI = MF.getFrameInfo();
2141 MachineRegisterInfo &MRI = MF.getRegInfo();
2142 SystemZMachineFunctionInfo *FuncInfo =
2143 MF.getInfo<SystemZMachineFunctionInfo>();
2144 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2145 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2146
2147 // Assign locations to all of the incoming arguments.
2148 SmallVector<CCValAssign, 16> ArgLocs;
2149 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2150 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
2151 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2152
2153 unsigned NumFixedGPRs = 0;
2154 unsigned NumFixedFPRs = 0;
2155 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2156 SDValue ArgValue;
2157 CCValAssign &VA = ArgLocs[I];
2158 EVT LocVT = VA.getLocVT();
2159 if (VA.isRegLoc()) {
2160 // Arguments passed in registers
2161 const TargetRegisterClass *RC;
2162 switch (LocVT.getSimpleVT().SimpleTy) {
2163 default:
2164 // Integers smaller than i64 should be promoted to i64.
2165 llvm_unreachable("Unexpected argument type");
2166 case MVT::i32:
2167 NumFixedGPRs += 1;
2168 RC = &SystemZ::GR32BitRegClass;
2169 break;
2170 case MVT::i64:
2171 NumFixedGPRs += 1;
2172 RC = &SystemZ::GR64BitRegClass;
2173 break;
2174 case MVT::f16:
2175 NumFixedFPRs += 1;
2176 RC = &SystemZ::FP16BitRegClass;
2177 break;
2178 case MVT::f32:
2179 NumFixedFPRs += 1;
2180 RC = &SystemZ::FP32BitRegClass;
2181 break;
2182 case MVT::f64:
2183 NumFixedFPRs += 1;
2184 RC = &SystemZ::FP64BitRegClass;
2185 break;
2186 case MVT::f128:
2187 NumFixedFPRs += 2;
2188 RC = &SystemZ::FP128BitRegClass;
2189 break;
2190 case MVT::v16i8:
2191 case MVT::v8i16:
2192 case MVT::v4i32:
2193 case MVT::v2i64:
2194 case MVT::v8f16:
2195 case MVT::v4f32:
2196 case MVT::v2f64:
2197 RC = &SystemZ::VR128BitRegClass;
2198 break;
2199 }
2200
2201 Register VReg = MRI.createVirtualRegister(RegClass: RC);
2202 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
2203 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
2204 } else {
2205 assert(VA.isMemLoc() && "Argument not register or memory");
2206
2207 // Create the frame index object for this incoming parameter.
2208 // FIXME: Pre-include call frame size in the offset, should not
2209 // need to manually add it here.
2210 int64_t ArgSPOffset = VA.getLocMemOffset();
2211 if (Subtarget.isTargetXPLINK64()) {
2212 auto &XPRegs =
2213 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2214 ArgSPOffset += XPRegs.getCallFrameSize();
2215 }
2216 int FI =
2217 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2218
2219 // Create the SelectionDAG nodes corresponding to a load
2220 // from this parameter. Unpromoted ints and floats are
2221 // passed as right-justified 8-byte values.
2222 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2223 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2224 VA.getLocVT() == MVT::f16) {
2225 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2226 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2227 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2228 }
2229 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2230 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2231 }
2232
2233 // Convert the value of the argument register into the value that's
2234 // being passed.
2235 if (VA.getLocInfo() == CCValAssign::Indirect) {
2236 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2237 PtrInfo: MachinePointerInfo()));
2238 // If the original argument was split (e.g. i128), we need
2239 // to load all parts of it here (using the same address).
2240 MVT PartVT;
2241 unsigned NumParts;
2242 if (analyzeArgSplit(Args: Ins, ArgLocs, I, PartVT, NumParts)) {
2243 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2244 ++I;
2245 CCValAssign &PartVA = ArgLocs[I];
2246 unsigned PartOffset = Ins[I].PartOffset;
2247 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2248 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2249 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2250 PtrInfo: MachinePointerInfo()));
2251 assert(PartOffset && "Offset should be non-zero.");
2252 }
2253 }
2254 } else if (Subtarget.isTargetXPLINK64() &&
2255 (VA.getLocInfo() == CCValAssign::SExt ||
2256 VA.getLocInfo() == CCValAssign::ZExt) &&
2257 Ins[I].ArgVT.isSimple()) {
2258 // Some prior z/OS compilers do not always perform the extension of
2259 // short integer arguments or pointers. To accommodate those, do not
2260 // rely on that extension by avoiding any AssertSext/AssertZext nodes by
2261 // directly truncating ArgValue to the original argument type.
2262 MVT OrigVT = Ins[I].ArgVT.getSimpleVT();
2263 InVals.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: ArgValue));
2264 } else
2265 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2266 }
2267
2268 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2269 // Save the number of non-varargs registers for later use by va_start, etc.
2270 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2271 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2272
2273 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2274 Subtarget.getSpecialRegisters());
2275
2276 // Likewise the address (in the form of a frame index) of where the
2277 // first stack vararg would be. The 1-byte size here is arbitrary.
2278 // FIXME: Pre-include call frame size in the offset, should not
2279 // need to manually add it here.
2280 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2281 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2282 FuncInfo->setVarArgsFrameIndex(FI);
2283 }
2284
2285 if (IsVarArg && Subtarget.isTargetELF()) {
2286 // Save the number of non-varargs registers for later use by va_start, etc.
2287 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2288 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2289
2290 // Likewise the address (in the form of a frame index) of where the
2291 // first stack vararg would be. The 1-byte size here is arbitrary.
2292 int64_t VarArgsOffset = CCInfo.getStackSize();
2293 FuncInfo->setVarArgsFrameIndex(
2294 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2295
2296 // ...and a similar frame index for the caller-allocated save area
2297 // that will be used to store the incoming registers.
2298 int64_t RegSaveOffset =
2299 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2300 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2301 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2302
2303 // Store the FPR varargs in the reserved frame slots. (We store the
2304 // GPRs as part of the prologue.)
2305 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2306 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2307 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2308 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2309 int FI =
2310 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2311 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2312 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2313 RC: &SystemZ::FP64BitRegClass);
2314 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2315 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2316 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2317 }
2318 // Join the stores, which are independent of one another.
2319 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2320 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2321 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2322 }
2323 }
2324
2325 return Chain;
2326}
2327
2328static bool canUseSiblingCall(const CCState &ArgCCInfo,
2329 SmallVectorImpl<CCValAssign> &ArgLocs,
2330 SmallVectorImpl<ISD::OutputArg> &Outs) {
2331 // Punt if there are any indirect or stack arguments, or if the call
2332 // needs the callee-saved argument register R6, or if the call uses
2333 // the callee-saved register arguments SwiftSelf and SwiftError.
2334 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2335 CCValAssign &VA = ArgLocs[I];
2336 if (VA.getLocInfo() == CCValAssign::Indirect)
2337 return false;
2338 if (!VA.isRegLoc())
2339 return false;
2340 Register Reg = VA.getLocReg();
2341 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2342 return false;
2343 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2344 return false;
2345 }
2346 return true;
2347}
2348
2349static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2350 unsigned Offset, bool LoadAdr = false) {
2351 MachineFunction &MF = DAG.getMachineFunction();
2352 Register ADAvReg = getOrCreateADAVirtualRegister(MF);
2353 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2354
2355 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2356 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2357
2358 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2359 if (!LoadAdr)
2360 Result = DAG.getLoad(
2361 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2362 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2363
2364 return Result;
2365}
2366
2367// ADA access using Global value
2368// Note: for functions, address of descriptor is returned
2369static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2370 EVT PtrVT) {
2371 unsigned ADAtype;
2372 bool LoadAddr = false;
2373 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2374 bool IsFunction =
2375 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2376 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2377
2378 if (IsFunction) {
2379 if (IsInternal) {
2380 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2381 LoadAddr = true;
2382 } else
2383 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2384 } else {
2385 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2386 }
2387 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2388
2389 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2390}
2391
2392static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2393 SDLoc &DL, SDValue &Chain) {
2394 unsigned ADADelta = 0; // ADA offset in desc.
2395 unsigned EPADelta = 8; // EPA offset in desc.
2396 MachineFunction &MF = DAG.getMachineFunction();
2397 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2398
2399 // XPLink calling convention.
2400 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2401 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2402 G->getGlobal()->hasPrivateLinkage());
2403 if (IsInternal) {
2404 Register ADAvReg = getOrCreateADAVirtualRegister(MF);
2405 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2406 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2407 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2408 return true;
2409 } else {
2410 SDValue GA = DAG.getTargetGlobalAddress(
2411 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2412 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2413 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2414 }
2415 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2416 SDValue ES = DAG.getTargetExternalSymbol(
2417 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2418 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2419 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2420 } else {
2421 // Function pointer case
2422 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2423 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2424 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2425 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2426 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2427 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2428 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2429 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2430 }
2431 return false;
2432}
2433
2434SDValue
2435SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2436 SmallVectorImpl<SDValue> &InVals) const {
2437 SelectionDAG &DAG = CLI.DAG;
2438 SDLoc &DL = CLI.DL;
2439 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2440 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2441 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2442 SDValue Chain = CLI.Chain;
2443 SDValue Callee = CLI.Callee;
2444 bool &IsTailCall = CLI.IsTailCall;
2445 CallingConv::ID CallConv = CLI.CallConv;
2446 bool IsVarArg = CLI.IsVarArg;
2447 MachineFunction &MF = DAG.getMachineFunction();
2448 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2449 LLVMContext &Ctx = *DAG.getContext();
2450 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2451
2452 // FIXME: z/OS support to be added in later.
2453 if (Subtarget.isTargetXPLINK64()) {
2454 if (CLI.CB && CLI.CB->isMustTailCall())
2455 report_fatal_error(reason: "musttail call is not supported on XPLINK64");
2456 IsTailCall = false;
2457 }
2458
2459 // Integer args <=32 bits should have an extension attribute.
2460 verifyNarrowIntegerArgs_Call(CLI, Outs, F: &MF.getFunction(), Callee);
2461
2462 // Analyze the operands of the call, assigning locations to each operand.
2463 SmallVector<CCValAssign, 16> ArgLocs;
2464 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2465 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2466
2467 // We don't support GuaranteedTailCallOpt, only automatically-detected
2468 // sibling calls.
2469 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2470 IsTailCall = false;
2471
2472 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
2473 report_fatal_error(reason: "failed to perform tail call elimination on a call "
2474 "site marked musttail");
2475
2476 // Get a count of how many bytes are to be pushed on the stack.
2477 unsigned NumBytes = ArgCCInfo.getStackSize();
2478
2479 // Mark the start of the call.
2480 if (!IsTailCall)
2481 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2482
2483 // Copy argument values to their designated locations.
2484 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2485 SmallVector<SDValue, 8> MemOpChains;
2486 SDValue StackPtr;
2487 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2488 CCValAssign &VA = ArgLocs[I];
2489 SDValue ArgValue = OutVals[I];
2490
2491 if (VA.getLocInfo() == CCValAssign::Indirect) {
2492 // Store the argument in a stack slot and pass its address.
2493 EVT SlotVT;
2494 MVT PartVT;
2495 unsigned NumParts = 1;
2496 if (analyzeArgSplit(Args: Outs, ArgLocs, I, PartVT, NumParts))
2497 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * NumParts);
2498 else
2499 SlotVT = Outs[I].VT;
2500 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2501 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2502
2503 MachinePointerInfo StackPtrInfo =
2504 MachinePointerInfo::getFixedStack(MF, FI);
2505 MemOpChains.push_back(
2506 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot, PtrInfo: StackPtrInfo));
2507 // If the original argument was split (e.g. i128), we need
2508 // to store all parts of it here (and pass just one address).
2509 assert(Outs[I].PartOffset == 0);
2510 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2511 ++I;
2512 SDValue PartValue = OutVals[I];
2513 unsigned PartOffset = Outs[I].PartOffset;
2514 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2515 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2516 MemOpChains.push_back(
2517 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2518 PtrInfo: StackPtrInfo.getWithOffset(O: PartOffset)));
2519 assert(PartOffset && "Offset should be non-zero.");
2520 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2521 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2522 }
2523 ArgValue = SpillSlot;
2524 } else
2525 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2526
2527 if (VA.isRegLoc()) {
2528 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2529 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2530 // and low values.
2531 if (VA.getLocVT() == MVT::i128)
2532 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2533 // Queue up the argument copies and emit them at the end.
2534 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2535 } else {
2536 assert(VA.isMemLoc() && "Argument not register or memory");
2537
2538 // Work out the address of the stack slot. Unpromoted ints and
2539 // floats are passed as right-justified 8-byte values.
2540 if (!StackPtr.getNode())
2541 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2542 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2543 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2544 VA.getLocMemOffset();
2545 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2546 Offset += 4;
2547 else if (VA.getLocVT() == MVT::f16)
2548 Offset += 6;
2549 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2550 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2551
2552 // Emit the store.
2553 MemOpChains.push_back(
2554 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2555
2556 // Although long doubles or vectors are passed through the stack when
2557 // they are vararg (non-fixed arguments), if a long double or vector
2558 // occupies the third and fourth slot of the argument list GPR3 should
2559 // still shadow the third slot of the argument list.
2560 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2561 SDValue ShadowArgValue =
2562 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2563 N2: DAG.getIntPtrConstant(Val: 1, DL));
2564 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2565 }
2566 }
2567 }
2568
2569 // Join the stores, which are independent of one another.
2570 if (!MemOpChains.empty())
2571 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2572
2573 // Accept direct calls by converting symbolic call addresses to the
2574 // associated Target* opcodes. Force %r1 to be used for indirect
2575 // tail calls.
2576 SDValue Glue;
2577
2578 if (Subtarget.isTargetXPLINK64()) {
2579 SDValue ADA;
2580 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2581 if (!IsBRASL) {
2582 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2583 ->getAddressOfCalleeRegister();
2584 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2585 Glue = Chain.getValue(R: 1);
2586 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2587 }
2588 RegsToPass.push_back(Elt: std::make_pair(
2589 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2590 } else {
2591 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2592 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2593 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2594 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2595 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2596 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2597 } else if (IsTailCall) {
2598 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2599 Glue = Chain.getValue(R: 1);
2600 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2601 }
2602 }
2603
2604 // Build a sequence of copy-to-reg nodes, chained and glued together.
2605 for (const auto &[Reg, N] : RegsToPass) {
2606 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2607 Glue = Chain.getValue(R: 1);
2608 }
2609
2610 // The first call operand is the chain and the second is the target address.
2611 SmallVector<SDValue, 8> Ops;
2612 Ops.push_back(Elt: Chain);
2613 Ops.push_back(Elt: Callee);
2614
2615 // Add argument registers to the end of the list so that they are
2616 // known live into the call.
2617 for (const auto &[Reg, N] : RegsToPass)
2618 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2619
2620 // Add a register mask operand representing the call-preserved registers.
2621 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2622 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2623 assert(Mask && "Missing call preserved mask for calling convention");
2624 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2625
2626 // Glue the call to the argument copies, if any.
2627 if (Glue.getNode())
2628 Ops.push_back(Elt: Glue);
2629
2630 // Emit the call.
2631 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2632 if (IsTailCall) {
2633 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2634 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2635 return Ret;
2636 }
2637 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2638 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2639 Glue = Chain.getValue(R: 1);
2640
2641 // Mark the end of the call, which is glued to the call itself.
2642 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2643 Glue = Chain.getValue(R: 1);
2644
2645 // Assign locations to each value returned by this call.
2646 SmallVector<CCValAssign, 16> RetLocs;
2647 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2648 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2649
2650 // Copy all of the result registers out of their specified physreg.
2651 for (CCValAssign &VA : RetLocs) {
2652 // Copy the value out, gluing the copy to the end of the call sequence.
2653 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2654 VT: VA.getLocVT(), Glue);
2655 Chain = RetValue.getValue(R: 1);
2656 Glue = RetValue.getValue(R: 2);
2657
2658 // Convert the value of the return register into the value that's
2659 // being returned.
2660 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2661 }
2662
2663 return Chain;
2664}
2665
2666// Generate a call taking the given operands as arguments and returning a
2667// result of type RetVT.
2668std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2669 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2670 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2671 bool DoesNotReturn, bool IsReturnValueUsed) const {
2672 TargetLowering::ArgListTy Args;
2673 Args.reserve(n: Ops.size());
2674
2675 for (SDValue Op : Ops) {
2676 TargetLowering::ArgListEntry Entry(
2677 Op, Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
2678 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2679 Entry.IsZExt = !Entry.IsSExt;
2680 Args.push_back(x: Entry);
2681 }
2682
2683 SDValue Callee =
2684 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2685
2686 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2687 TargetLowering::CallLoweringInfo CLI(DAG);
2688 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2689 CLI.setDebugLoc(DL)
2690 .setChain(Chain)
2691 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2692 .setNoReturn(DoesNotReturn)
2693 .setDiscardResult(!IsReturnValueUsed)
2694 .setSExtResult(SignExtend)
2695 .setZExtResult(!SignExtend);
2696 return LowerCallTo(CLI);
2697}
2698
2699bool SystemZTargetLowering::CanLowerReturn(
2700 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2701 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2702 const Type *RetTy) const {
2703 // Special case that we cannot easily detect in RetCC_SystemZ since
2704 // i128 may not be a legal type.
2705 for (auto &Out : Outs)
2706 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2707 return false;
2708
2709 SmallVector<CCValAssign, 16> RetLocs;
2710 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2711 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2712}
2713
2714SDValue
2715SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2716 bool IsVarArg,
2717 const SmallVectorImpl<ISD::OutputArg> &Outs,
2718 const SmallVectorImpl<SDValue> &OutVals,
2719 const SDLoc &DL, SelectionDAG &DAG) const {
2720 MachineFunction &MF = DAG.getMachineFunction();
2721
2722 // Integer args <=32 bits should have an extension attribute.
2723 verifyNarrowIntegerArgs_Ret(CallConv, Outs, F: &MF.getFunction());
2724
2725 // Assign locations to each returned value.
2726 SmallVector<CCValAssign, 16> RetLocs;
2727 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2728 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2729
2730 // Quick exit for void returns
2731 if (RetLocs.empty())
2732 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2733
2734 if (CallConv == CallingConv::GHC)
2735 report_fatal_error(reason: "GHC functions return void only");
2736
2737 // Copy the result values into the output registers.
2738 SDValue Glue;
2739 SmallVector<SDValue, 4> RetOps;
2740 RetOps.push_back(Elt: Chain);
2741 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2742 CCValAssign &VA = RetLocs[I];
2743 SDValue RetValue = OutVals[I];
2744
2745 // Make the return register live on exit.
2746 assert(VA.isRegLoc() && "Can only return in registers!");
2747
2748 // Promote the value as required.
2749 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2750
2751 // Chain and glue the copies together.
2752 Register Reg = VA.getLocReg();
2753 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2754 Glue = Chain.getValue(R: 1);
2755 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2756 }
2757
2758 // Update chain and glue.
2759 RetOps[0] = Chain;
2760 if (Glue.getNode())
2761 RetOps.push_back(Elt: Glue);
2762
2763 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2764}
2765
2766// Return true if Op is an intrinsic node with chain that returns the CC value
2767// as its only (other) argument. Provide the associated SystemZISD opcode and
2768// the mask of valid CC values if so.
2769static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2770 unsigned &CCValid) {
2771 unsigned Id = Op.getConstantOperandVal(i: 1);
2772 switch (Id) {
2773 case Intrinsic::s390_tbegin:
2774 Opcode = SystemZISD::TBEGIN;
2775 CCValid = SystemZ::CCMASK_TBEGIN;
2776 return true;
2777
2778 case Intrinsic::s390_tbegin_nofloat:
2779 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2780 CCValid = SystemZ::CCMASK_TBEGIN;
2781 return true;
2782
2783 case Intrinsic::s390_tend:
2784 Opcode = SystemZISD::TEND;
2785 CCValid = SystemZ::CCMASK_TEND;
2786 return true;
2787
2788 default:
2789 return false;
2790 }
2791}
2792
2793// Return true if Op is an intrinsic node without chain that returns the
2794// CC value as its final argument. Provide the associated SystemZISD
2795// opcode and the mask of valid CC values if so.
2796static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2797 unsigned Id = Op.getConstantOperandVal(i: 0);
2798 switch (Id) {
2799 case Intrinsic::s390_vpkshs:
2800 case Intrinsic::s390_vpksfs:
2801 case Intrinsic::s390_vpksgs:
2802 Opcode = SystemZISD::PACKS_CC;
2803 CCValid = SystemZ::CCMASK_VCMP;
2804 return true;
2805
2806 case Intrinsic::s390_vpklshs:
2807 case Intrinsic::s390_vpklsfs:
2808 case Intrinsic::s390_vpklsgs:
2809 Opcode = SystemZISD::PACKLS_CC;
2810 CCValid = SystemZ::CCMASK_VCMP;
2811 return true;
2812
2813 case Intrinsic::s390_vceqbs:
2814 case Intrinsic::s390_vceqhs:
2815 case Intrinsic::s390_vceqfs:
2816 case Intrinsic::s390_vceqgs:
2817 case Intrinsic::s390_vceqqs:
2818 Opcode = SystemZISD::VICMPES;
2819 CCValid = SystemZ::CCMASK_VCMP;
2820 return true;
2821
2822 case Intrinsic::s390_vchbs:
2823 case Intrinsic::s390_vchhs:
2824 case Intrinsic::s390_vchfs:
2825 case Intrinsic::s390_vchgs:
2826 case Intrinsic::s390_vchqs:
2827 Opcode = SystemZISD::VICMPHS;
2828 CCValid = SystemZ::CCMASK_VCMP;
2829 return true;
2830
2831 case Intrinsic::s390_vchlbs:
2832 case Intrinsic::s390_vchlhs:
2833 case Intrinsic::s390_vchlfs:
2834 case Intrinsic::s390_vchlgs:
2835 case Intrinsic::s390_vchlqs:
2836 Opcode = SystemZISD::VICMPHLS;
2837 CCValid = SystemZ::CCMASK_VCMP;
2838 return true;
2839
2840 case Intrinsic::s390_vtm:
2841 Opcode = SystemZISD::VTM;
2842 CCValid = SystemZ::CCMASK_VCMP;
2843 return true;
2844
2845 case Intrinsic::s390_vfaebs:
2846 case Intrinsic::s390_vfaehs:
2847 case Intrinsic::s390_vfaefs:
2848 Opcode = SystemZISD::VFAE_CC;
2849 CCValid = SystemZ::CCMASK_ANY;
2850 return true;
2851
2852 case Intrinsic::s390_vfaezbs:
2853 case Intrinsic::s390_vfaezhs:
2854 case Intrinsic::s390_vfaezfs:
2855 Opcode = SystemZISD::VFAEZ_CC;
2856 CCValid = SystemZ::CCMASK_ANY;
2857 return true;
2858
2859 case Intrinsic::s390_vfeebs:
2860 case Intrinsic::s390_vfeehs:
2861 case Intrinsic::s390_vfeefs:
2862 Opcode = SystemZISD::VFEE_CC;
2863 CCValid = SystemZ::CCMASK_ANY;
2864 return true;
2865
2866 case Intrinsic::s390_vfeezbs:
2867 case Intrinsic::s390_vfeezhs:
2868 case Intrinsic::s390_vfeezfs:
2869 Opcode = SystemZISD::VFEEZ_CC;
2870 CCValid = SystemZ::CCMASK_ANY;
2871 return true;
2872
2873 case Intrinsic::s390_vfenebs:
2874 case Intrinsic::s390_vfenehs:
2875 case Intrinsic::s390_vfenefs:
2876 Opcode = SystemZISD::VFENE_CC;
2877 CCValid = SystemZ::CCMASK_ANY;
2878 return true;
2879
2880 case Intrinsic::s390_vfenezbs:
2881 case Intrinsic::s390_vfenezhs:
2882 case Intrinsic::s390_vfenezfs:
2883 Opcode = SystemZISD::VFENEZ_CC;
2884 CCValid = SystemZ::CCMASK_ANY;
2885 return true;
2886
2887 case Intrinsic::s390_vistrbs:
2888 case Intrinsic::s390_vistrhs:
2889 case Intrinsic::s390_vistrfs:
2890 Opcode = SystemZISD::VISTR_CC;
2891 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2892 return true;
2893
2894 case Intrinsic::s390_vstrcbs:
2895 case Intrinsic::s390_vstrchs:
2896 case Intrinsic::s390_vstrcfs:
2897 Opcode = SystemZISD::VSTRC_CC;
2898 CCValid = SystemZ::CCMASK_ANY;
2899 return true;
2900
2901 case Intrinsic::s390_vstrczbs:
2902 case Intrinsic::s390_vstrczhs:
2903 case Intrinsic::s390_vstrczfs:
2904 Opcode = SystemZISD::VSTRCZ_CC;
2905 CCValid = SystemZ::CCMASK_ANY;
2906 return true;
2907
2908 case Intrinsic::s390_vstrsb:
2909 case Intrinsic::s390_vstrsh:
2910 case Intrinsic::s390_vstrsf:
2911 Opcode = SystemZISD::VSTRS_CC;
2912 CCValid = SystemZ::CCMASK_ANY;
2913 return true;
2914
2915 case Intrinsic::s390_vstrszb:
2916 case Intrinsic::s390_vstrszh:
2917 case Intrinsic::s390_vstrszf:
2918 Opcode = SystemZISD::VSTRSZ_CC;
2919 CCValid = SystemZ::CCMASK_ANY;
2920 return true;
2921
2922 case Intrinsic::s390_vfcedbs:
2923 case Intrinsic::s390_vfcesbs:
2924 Opcode = SystemZISD::VFCMPES;
2925 CCValid = SystemZ::CCMASK_VCMP;
2926 return true;
2927
2928 case Intrinsic::s390_vfchdbs:
2929 case Intrinsic::s390_vfchsbs:
2930 Opcode = SystemZISD::VFCMPHS;
2931 CCValid = SystemZ::CCMASK_VCMP;
2932 return true;
2933
2934 case Intrinsic::s390_vfchedbs:
2935 case Intrinsic::s390_vfchesbs:
2936 Opcode = SystemZISD::VFCMPHES;
2937 CCValid = SystemZ::CCMASK_VCMP;
2938 return true;
2939
2940 case Intrinsic::s390_vftcidb:
2941 case Intrinsic::s390_vftcisb:
2942 Opcode = SystemZISD::VFTCI;
2943 CCValid = SystemZ::CCMASK_VCMP;
2944 return true;
2945
2946 case Intrinsic::s390_tdc:
2947 Opcode = SystemZISD::TDC;
2948 CCValid = SystemZ::CCMASK_TDC;
2949 return true;
2950
2951 default:
2952 return false;
2953 }
2954}
2955
2956// Emit an intrinsic with chain and an explicit CC register result.
2957static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2958 unsigned Opcode) {
2959 // Copy all operands except the intrinsic ID.
2960 unsigned NumOps = Op.getNumOperands();
2961 SmallVector<SDValue, 6> Ops;
2962 Ops.reserve(N: NumOps - 1);
2963 Ops.push_back(Elt: Op.getOperand(i: 0));
2964 for (unsigned I = 2; I < NumOps; ++I)
2965 Ops.push_back(Elt: Op.getOperand(i: I));
2966
2967 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2968 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2969 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2970 SDValue OldChain = SDValue(Op.getNode(), 1);
2971 SDValue NewChain = SDValue(Intr.getNode(), 1);
2972 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2973 return Intr.getNode();
2974}
2975
2976// Emit an intrinsic with an explicit CC register result.
2977static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2978 unsigned Opcode) {
2979 // Copy all operands except the intrinsic ID.
2980 SDLoc DL(Op);
2981 unsigned NumOps = Op.getNumOperands();
2982 SmallVector<SDValue, 6> Ops;
2983 Ops.reserve(N: NumOps - 1);
2984 for (unsigned I = 1; I < NumOps; ++I) {
2985 SDValue CurrOper = Op.getOperand(i: I);
2986 if (CurrOper.getValueType() == MVT::f16) {
2987 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2988 "Unhandled intrinsic with f16 operand.");
2989 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2990 }
2991 Ops.push_back(Elt: CurrOper);
2992 }
2993
2994 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2995 return Intr.getNode();
2996}
2997
2998// CC is a comparison that will be implemented using an integer or
2999// floating-point comparison. Return the condition code mask for
3000// a branch on true. In the integer case, CCMASK_CMP_UO is set for
3001// unsigned comparisons and clear for signed ones. In the floating-point
3002// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
3003static unsigned CCMaskForCondCode(ISD::CondCode CC) {
3004#define CONV(X) \
3005 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
3006 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
3007 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
3008
3009 switch (CC) {
3010 default:
3011 llvm_unreachable("Invalid integer condition!");
3012
3013 CONV(EQ);
3014 CONV(NE);
3015 CONV(GT);
3016 CONV(GE);
3017 CONV(LT);
3018 CONV(LE);
3019
3020 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
3021 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
3022 }
3023#undef CONV
3024}
3025
3026// If C can be converted to a comparison against zero, adjust the operands
3027// as necessary.
3028static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3029 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3030 return;
3031
3032 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
3033 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
3034 return;
3035
3036 int64_t Value = ConstOp1->getSExtValue();
3037 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
3038 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
3039 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
3040 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
3041 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3042 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
3043 }
3044}
3045
3046// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
3047// adjust the operands as necessary.
3048static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
3049 Comparison &C) {
3050 // For us to make any changes, it must a comparison between a single-use
3051 // load and a constant.
3052 if (!C.Op0.hasOneUse() ||
3053 C.Op0.getOpcode() != ISD::LOAD ||
3054 C.Op1.getOpcode() != ISD::Constant)
3055 return;
3056
3057 // We must have an 8- or 16-bit load.
3058 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
3059 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
3060 if ((NumBits != 8 && NumBits != 16) ||
3061 NumBits != Load->getMemoryVT().getStoreSizeInBits())
3062 return;
3063
3064 // The load must be an extending one and the constant must be within the
3065 // range of the unextended value.
3066 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
3067 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
3068 return;
3069 uint64_t Value = ConstOp1->getZExtValue();
3070 uint64_t Mask = (1 << NumBits) - 1;
3071 if (Load->getExtensionType() == ISD::SEXTLOAD) {
3072 // Make sure that ConstOp1 is in range of C.Op0.
3073 int64_t SignedValue = ConstOp1->getSExtValue();
3074 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
3075 return;
3076 if (C.ICmpType != SystemZICMP::SignedOnly) {
3077 // Unsigned comparison between two sign-extended values is equivalent
3078 // to unsigned comparison between two zero-extended values.
3079 Value &= Mask;
3080 } else if (NumBits == 8) {
3081 // Try to treat the comparison as unsigned, so that we can use CLI.
3082 // Adjust CCMask and Value as necessary.
3083 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
3084 // Test whether the high bit of the byte is set.
3085 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
3086 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
3087 // Test whether the high bit of the byte is clear.
3088 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
3089 else
3090 // No instruction exists for this combination.
3091 return;
3092 C.ICmpType = SystemZICMP::UnsignedOnly;
3093 }
3094 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
3095 if (Value > Mask)
3096 return;
3097 // If the constant is in range, we can use any comparison.
3098 C.ICmpType = SystemZICMP::Any;
3099 } else
3100 return;
3101
3102 // Make sure that the first operand is an i32 of the right extension type.
3103 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
3104 ISD::SEXTLOAD :
3105 ISD::ZEXTLOAD);
3106 if (C.Op0.getValueType() != MVT::i32 ||
3107 Load->getExtensionType() != ExtType) {
3108 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
3109 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
3110 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
3111 MMOFlags: Load->getMemOperand()->getFlags());
3112 // Update the chain uses.
3113 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
3114 }
3115
3116 // Make sure that the second operand is an i32 with the right value.
3117 if (C.Op1.getValueType() != MVT::i32 ||
3118 Value != ConstOp1->getZExtValue())
3119 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
3120}
3121
3122// Return true if Op is either an unextended load, or a load suitable
3123// for integer register-memory comparisons of type ICmpType.
3124static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3125 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
3126 if (Load) {
3127 // There are no instructions to compare a register with a memory byte.
3128 if (Load->getMemoryVT() == MVT::i8)
3129 return false;
3130 // Otherwise decide on extension type.
3131 switch (Load->getExtensionType()) {
3132 case ISD::NON_EXTLOAD:
3133 return true;
3134 case ISD::SEXTLOAD:
3135 return ICmpType != SystemZICMP::UnsignedOnly;
3136 case ISD::ZEXTLOAD:
3137 return ICmpType != SystemZICMP::SignedOnly;
3138 default:
3139 break;
3140 }
3141 }
3142 return false;
3143}
3144
3145// Return true if it is better to swap the operands of C.
3146static bool shouldSwapCmpOperands(const Comparison &C) {
3147 // If one side of the compare is a load of the stackguard reference value,
3148 // then that load should be Op1.
3149 if (C.Op0.isMachineOpcode() &&
3150 (C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3151 return true;
3152
3153 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3154 if (C.Op0.getValueType() == MVT::i128)
3155 return false;
3156 if (C.Op0.getValueType() == MVT::f128)
3157 return false;
3158
3159 // Always keep a floating-point constant second, since comparisons with
3160 // zero can use LOAD TEST and comparisons with other constants make a
3161 // natural memory operand.
3162 if (isa<ConstantFPSDNode>(Val: C.Op1))
3163 return false;
3164
3165 // Never swap comparisons with zero since there are many ways to optimize
3166 // those later.
3167 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
3168 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3169 return false;
3170
3171 // Also keep natural memory operands second if the loaded value is
3172 // only used here. Several comparisons have memory forms.
3173 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
3174 return false;
3175
3176 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3177 // In that case we generally prefer the memory to be second.
3178 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
3179 // The only exceptions are when the second operand is a constant and
3180 // we can use things like CHHSI.
3181 if (!ConstOp1)
3182 return true;
3183 // The unsigned memory-immediate instructions can handle 16-bit
3184 // unsigned integers.
3185 if (C.ICmpType != SystemZICMP::SignedOnly &&
3186 isUInt<16>(x: ConstOp1->getZExtValue()))
3187 return false;
3188 // The signed memory-immediate instructions can handle 16-bit
3189 // signed integers.
3190 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3191 isInt<16>(x: ConstOp1->getSExtValue()))
3192 return false;
3193 return true;
3194 }
3195
3196 // Try to promote the use of CGFR and CLGFR.
3197 unsigned Opcode0 = C.Op0.getOpcode();
3198 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3199 return true;
3200 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3201 return true;
3202 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3203 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
3204 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
3205 return true;
3206
3207 return false;
3208}
3209
3210// Check whether C tests for equality between X and Y and whether X - Y
3211// or Y - X is also computed. In that case it's better to compare the
3212// result of the subtraction against zero.
3213static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
3214 Comparison &C) {
3215 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3216 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3217 for (SDNode *N : C.Op0->users()) {
3218 if (N->getOpcode() == ISD::SUB &&
3219 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3220 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3221 // Disable the nsw and nuw flags: the backend needs to handle
3222 // overflow as well during comparison elimination.
3223 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3224 C.Op0 = SDValue(N, 0);
3225 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3226 return;
3227 }
3228 }
3229 }
3230}
3231
3232// Check whether C compares a floating-point value with zero and if that
3233// floating-point value is also negated. In this case we can use the
3234// negation to set CC, so avoiding separate LOAD AND TEST and
3235// LOAD (NEGATIVE/COMPLEMENT) instructions.
3236static void adjustForFNeg(Comparison &C) {
3237 // This optimization is invalid for strict comparisons, since FNEG
3238 // does not raise any exceptions.
3239 if (C.Chain)
3240 return;
3241 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3242 if (C1 && C1->isZero()) {
3243 for (SDNode *N : C.Op0->users()) {
3244 if (N->getOpcode() == ISD::FNEG) {
3245 C.Op0 = SDValue(N, 0);
3246 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3247 return;
3248 }
3249 }
3250 }
3251}
3252
3253// Check whether C compares (shl X, 32) with 0 and whether X is
3254// also sign-extended. In that case it is better to test the result
3255// of the sign extension using LTGFR.
3256//
3257// This case is important because InstCombine transforms a comparison
3258// with (sext (trunc X)) into a comparison with (shl X, 32).
3259static void adjustForLTGFR(Comparison &C) {
3260 // Check for a comparison between (shl X, 32) and 0.
3261 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3262 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3263 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3264 if (C1 && C1->getZExtValue() == 32) {
3265 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3266 // See whether X has any SIGN_EXTEND_INREG uses.
3267 for (SDNode *N : ShlOp0->users()) {
3268 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3269 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3270 C.Op0 = SDValue(N, 0);
3271 return;
3272 }
3273 }
3274 }
3275 }
3276}
3277
3278// If C compares the truncation of an extending load, try to compare
3279// the untruncated value instead. This exposes more opportunities to
3280// reuse CC.
3281static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3282 Comparison &C) {
3283 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3284 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3285 C.Op1.getOpcode() == ISD::Constant &&
3286 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3287 C.Op1->getAsZExtVal() == 0) {
3288 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3289 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3290 C.Op0.getValueSizeInBits().getFixedValue()) {
3291 unsigned Type = L->getExtensionType();
3292 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3293 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3294 C.Op0 = C.Op0.getOperand(i: 0);
3295 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3296 }
3297 }
3298 }
3299}
3300
3301// Adjust if a given Compare is a check of the stack guard against a stack
3302// guard instance on the stack. Specifically, this checks if:
3303// - The operands are a load of the stack guard, and a load from a stack slot
3304// - The original opcode is ICMP
3305// - ICMPType is compatible with unsigned comparison.
3306static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL,
3307 Comparison &C) {
3308
3309 // Opcode must be ICMP.
3310 if (C.Opcode != SystemZISD::ICMP)
3311 return;
3312 // ICmpType must be Unsigned or Any.
3313 if (C.ICmpType == SystemZICMP::SignedOnly)
3314 return;
3315 // Op0 must be FrameIndex Load.
3316 if (!(ISD::isNormalLoad(N: C.Op0.getNode()) &&
3317 dyn_cast<FrameIndexSDNode>(Val: C.Op0.getOperand(i: 1))))
3318 return;
3319 // Op1 must be LOAD_STACK_GUARD.
3320 if (!C.Op1.isMachineOpcode() ||
3321 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3322 return;
3323
3324 // At this point we are sure that this is a proper CMP_STACKGUARD
3325 // case, update the opcode to reflect this.
3326 C.Opcode = SystemZISD::CMP_STACKGUARD;
3327 C.Op1 = SDValue();
3328}
3329
3330// Return true if shift operation N has an in-range constant shift value.
3331// Store it in ShiftVal if so.
3332static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3333 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3334 if (!Shift)
3335 return false;
3336
3337 uint64_t Amount = Shift->getZExtValue();
3338 if (Amount >= N.getValueSizeInBits())
3339 return false;
3340
3341 ShiftVal = Amount;
3342 return true;
3343}
3344
3345// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3346// instruction and whether the CC value is descriptive enough to handle
3347// a comparison of type Opcode between the AND result and CmpVal.
3348// CCMask says which comparison result is being tested and BitSize is
3349// the number of bits in the operands. If TEST UNDER MASK can be used,
3350// return the corresponding CC mask, otherwise return 0.
3351static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3352 uint64_t Mask, uint64_t CmpVal,
3353 unsigned ICmpType) {
3354 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3355
3356 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3357 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3358 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3359 return 0;
3360
3361 // Work out the masks for the lowest and highest bits.
3362 uint64_t High = llvm::bit_floor(Value: Mask);
3363 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3364
3365 // Signed ordered comparisons are effectively unsigned if the sign
3366 // bit is dropped.
3367 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3368
3369 // Check for equality comparisons with 0, or the equivalent.
3370 if (CmpVal == 0) {
3371 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3372 return SystemZ::CCMASK_TM_ALL_0;
3373 if (CCMask == SystemZ::CCMASK_CMP_NE)
3374 return SystemZ::CCMASK_TM_SOME_1;
3375 }
3376 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3377 if (CCMask == SystemZ::CCMASK_CMP_LT)
3378 return SystemZ::CCMASK_TM_ALL_0;
3379 if (CCMask == SystemZ::CCMASK_CMP_GE)
3380 return SystemZ::CCMASK_TM_SOME_1;
3381 }
3382 if (EffectivelyUnsigned && CmpVal < Low) {
3383 if (CCMask == SystemZ::CCMASK_CMP_LE)
3384 return SystemZ::CCMASK_TM_ALL_0;
3385 if (CCMask == SystemZ::CCMASK_CMP_GT)
3386 return SystemZ::CCMASK_TM_SOME_1;
3387 }
3388
3389 // Check for equality comparisons with the mask, or the equivalent.
3390 if (CmpVal == Mask) {
3391 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3392 return SystemZ::CCMASK_TM_ALL_1;
3393 if (CCMask == SystemZ::CCMASK_CMP_NE)
3394 return SystemZ::CCMASK_TM_SOME_0;
3395 }
3396 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3397 if (CCMask == SystemZ::CCMASK_CMP_GT)
3398 return SystemZ::CCMASK_TM_ALL_1;
3399 if (CCMask == SystemZ::CCMASK_CMP_LE)
3400 return SystemZ::CCMASK_TM_SOME_0;
3401 }
3402 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3403 if (CCMask == SystemZ::CCMASK_CMP_GE)
3404 return SystemZ::CCMASK_TM_ALL_1;
3405 if (CCMask == SystemZ::CCMASK_CMP_LT)
3406 return SystemZ::CCMASK_TM_SOME_0;
3407 }
3408
3409 // Check for ordered comparisons with the top bit.
3410 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3411 if (CCMask == SystemZ::CCMASK_CMP_LE)
3412 return SystemZ::CCMASK_TM_MSB_0;
3413 if (CCMask == SystemZ::CCMASK_CMP_GT)
3414 return SystemZ::CCMASK_TM_MSB_1;
3415 }
3416 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3417 if (CCMask == SystemZ::CCMASK_CMP_LT)
3418 return SystemZ::CCMASK_TM_MSB_0;
3419 if (CCMask == SystemZ::CCMASK_CMP_GE)
3420 return SystemZ::CCMASK_TM_MSB_1;
3421 }
3422
3423 // If there are just two bits, we can do equality checks for Low and High
3424 // as well.
3425 if (Mask == Low + High) {
3426 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3427 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3428 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3429 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3430 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3431 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3432 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3433 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3434 }
3435
3436 // Looks like we've exhausted our options.
3437 return 0;
3438}
3439
3440// See whether C can be implemented as a TEST UNDER MASK instruction.
3441// Update the arguments with the TM version if so.
3442static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3443 Comparison &C) {
3444 // Use VECTOR TEST UNDER MASK for i128 operations.
3445 if (C.Op0.getValueType() == MVT::i128) {
3446 // We can use VTM for EQ/NE comparisons of x & y against 0.
3447 if (C.Op0.getOpcode() == ISD::AND &&
3448 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3449 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3450 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3451 if (Mask && Mask->getAPIntValue() == 0) {
3452 C.Opcode = SystemZISD::VTM;
3453 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3454 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3455 C.CCValid = SystemZ::CCMASK_VCMP;
3456 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3457 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3458 else
3459 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3460 }
3461 }
3462 return;
3463 }
3464
3465 // Check that we have a comparison with a constant.
3466 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3467 if (!ConstOp1)
3468 return;
3469 uint64_t CmpVal = ConstOp1->getZExtValue();
3470
3471 // Check whether the nonconstant input is an AND with a constant mask.
3472 Comparison NewC(C);
3473 uint64_t MaskVal;
3474 ConstantSDNode *Mask = nullptr;
3475 if (C.Op0.getOpcode() == ISD::AND) {
3476 NewC.Op0 = C.Op0.getOperand(i: 0);
3477 NewC.Op1 = C.Op0.getOperand(i: 1);
3478 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3479 if (!Mask)
3480 return;
3481 MaskVal = Mask->getZExtValue();
3482 } else {
3483 // There is no instruction to compare with a 64-bit immediate
3484 // so use TMHH instead if possible. We need an unsigned ordered
3485 // comparison with an i64 immediate.
3486 if (NewC.Op0.getValueType() != MVT::i64 ||
3487 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3488 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3489 NewC.ICmpType == SystemZICMP::SignedOnly)
3490 return;
3491 // Convert LE and GT comparisons into LT and GE.
3492 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3493 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3494 if (CmpVal == uint64_t(-1))
3495 return;
3496 CmpVal += 1;
3497 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3498 }
3499 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3500 // be masked off without changing the result.
3501 MaskVal = -(CmpVal & -CmpVal);
3502 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3503 }
3504 if (!MaskVal)
3505 return;
3506
3507 // Check whether the combination of mask, comparison value and comparison
3508 // type are suitable.
3509 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3510 unsigned NewCCMask, ShiftVal;
3511 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3512 NewC.Op0.getOpcode() == ISD::SHL &&
3513 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3514 (MaskVal >> ShiftVal != 0) &&
3515 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3516 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3517 Mask: MaskVal >> ShiftVal,
3518 CmpVal: CmpVal >> ShiftVal,
3519 ICmpType: SystemZICMP::Any))) {
3520 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3521 MaskVal >>= ShiftVal;
3522 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3523 NewC.Op0.getOpcode() == ISD::SRL &&
3524 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3525 (MaskVal << ShiftVal != 0) &&
3526 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3527 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3528 Mask: MaskVal << ShiftVal,
3529 CmpVal: CmpVal << ShiftVal,
3530 ICmpType: SystemZICMP::UnsignedOnly))) {
3531 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3532 MaskVal <<= ShiftVal;
3533 } else {
3534 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3535 ICmpType: NewC.ICmpType);
3536 if (!NewCCMask)
3537 return;
3538 }
3539
3540 // Go ahead and make the change.
3541 C.Opcode = SystemZISD::TM;
3542 C.Op0 = NewC.Op0;
3543 if (Mask && Mask->getZExtValue() == MaskVal)
3544 C.Op1 = SDValue(Mask, 0);
3545 else
3546 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3547 C.CCValid = SystemZ::CCMASK_TM;
3548 C.CCMask = NewCCMask;
3549}
3550
3551// Implement i128 comparison in vector registers.
3552static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3553 Comparison &C) {
3554 if (C.Opcode != SystemZISD::ICMP)
3555 return;
3556 if (C.Op0.getValueType() != MVT::i128)
3557 return;
3558
3559 // Recognize vector comparison reductions.
3560 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3561 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3562 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3563 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3564 bool CmpNull = isNullConstant(V: C.Op1);
3565 SDValue Src = peekThroughBitcasts(V: C.Op0);
3566 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3567 Src = Src.getOperand(i: 0);
3568 CmpNull = !CmpNull;
3569 }
3570 unsigned Opcode = 0;
3571 if (Src.hasOneUse()) {
3572 switch (Src.getOpcode()) {
3573 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3574 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3575 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3576 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3577 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3578 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3579 default: break;
3580 }
3581 }
3582 if (Opcode) {
3583 C.Opcode = Opcode;
3584 C.Op0 = Src->getOperand(Num: 0);
3585 C.Op1 = Src->getOperand(Num: 1);
3586 C.CCValid = SystemZ::CCMASK_VCMP;
3587 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3588 if (!CmpEq)
3589 C.CCMask ^= C.CCValid;
3590 return;
3591 }
3592 }
3593
3594 // Everything below here is not useful if we have native i128 compares.
3595 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3596 return;
3597
3598 // (In-)Equality comparisons can be implemented via VCEQGS.
3599 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3600 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3601 C.Opcode = SystemZISD::VICMPES;
3602 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3603 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3604 C.CCValid = SystemZ::CCMASK_VCMP;
3605 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3606 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3607 else
3608 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3609 return;
3610 }
3611
3612 // Normalize other comparisons to GT.
3613 bool Swap = false, Invert = false;
3614 switch (C.CCMask) {
3615 case SystemZ::CCMASK_CMP_GT: break;
3616 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3617 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3618 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3619 default: llvm_unreachable("Invalid integer condition!");
3620 }
3621 if (Swap)
3622 std::swap(a&: C.Op0, b&: C.Op1);
3623
3624 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3625 C.Opcode = SystemZISD::UCMP128HI;
3626 else
3627 C.Opcode = SystemZISD::SCMP128HI;
3628 C.CCValid = SystemZ::CCMASK_ANY;
3629 C.CCMask = SystemZ::CCMASK_1;
3630
3631 if (Invert)
3632 C.CCMask ^= C.CCValid;
3633}
3634
3635// See whether the comparison argument contains a redundant AND
3636// and remove it if so. This sometimes happens due to the generic
3637// BRCOND expansion.
3638static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3639 Comparison &C) {
3640 if (C.Op0.getOpcode() != ISD::AND)
3641 return;
3642 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3643 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3644 return;
3645 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3646 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3647 return;
3648
3649 C.Op0 = C.Op0.getOperand(i: 0);
3650}
3651
3652// Return a Comparison that tests the condition-code result of intrinsic
3653// node Call against constant integer CC using comparison code Cond.
3654// Opcode is the opcode of the SystemZISD operation for the intrinsic
3655// and CCValid is the set of possible condition-code results.
3656static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3657 SDValue Call, unsigned CCValid, uint64_t CC,
3658 ISD::CondCode Cond) {
3659 Comparison C(Call, SDValue(), SDValue());
3660 C.Opcode = Opcode;
3661 C.CCValid = CCValid;
3662 if (Cond == ISD::SETEQ)
3663 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3664 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3665 else if (Cond == ISD::SETNE)
3666 // ...and the inverse of that.
3667 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3668 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3669 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3670 // always true for CC>3.
3671 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3672 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3673 // ...and the inverse of that.
3674 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3675 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3676 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3677 // always true for CC>3.
3678 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3679 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3680 // ...and the inverse of that.
3681 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3682 else
3683 llvm_unreachable("Unexpected integer comparison type");
3684 C.CCMask &= CCValid;
3685 return C;
3686}
3687
3688// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3689static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3690 ISD::CondCode Cond, const SDLoc &DL,
3691 SDValue Chain = SDValue(),
3692 bool IsSignaling = false) {
3693 if (CmpOp1.getOpcode() == ISD::Constant) {
3694 assert(!Chain);
3695 unsigned Opcode, CCValid;
3696 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3697 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3698 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3699 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3700 CC: CmpOp1->getAsZExtVal(), Cond);
3701 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3702 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3703 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3704 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3705 CC: CmpOp1->getAsZExtVal(), Cond);
3706 }
3707 Comparison C(CmpOp0, CmpOp1, Chain);
3708 C.CCMask = CCMaskForCondCode(CC: Cond);
3709 if (C.Op0.getValueType().isFloatingPoint()) {
3710 C.CCValid = SystemZ::CCMASK_FCMP;
3711 if (!C.Chain)
3712 C.Opcode = SystemZISD::FCMP;
3713 else if (!IsSignaling)
3714 C.Opcode = SystemZISD::STRICT_FCMP;
3715 else
3716 C.Opcode = SystemZISD::STRICT_FCMPS;
3717 adjustForFNeg(C);
3718 } else {
3719 assert(!C.Chain);
3720 C.CCValid = SystemZ::CCMASK_ICMP;
3721 C.Opcode = SystemZISD::ICMP;
3722 // Choose the type of comparison. Equality and inequality tests can
3723 // use either signed or unsigned comparisons. The choice also doesn't
3724 // matter if both sign bits are known to be clear. In those cases we
3725 // want to give the main isel code the freedom to choose whichever
3726 // form fits best.
3727 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3728 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3729 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3730 C.ICmpType = SystemZICMP::Any;
3731 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3732 C.ICmpType = SystemZICMP::UnsignedOnly;
3733 else
3734 C.ICmpType = SystemZICMP::SignedOnly;
3735 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3736 adjustForRedundantAnd(DAG, DL, C);
3737 adjustZeroCmp(DAG, DL, C);
3738 adjustSubwordCmp(DAG, DL, C);
3739 adjustForSubtraction(DAG, DL, C);
3740 adjustForLTGFR(C);
3741 adjustICmpTruncate(DAG, DL, C);
3742 }
3743
3744 if (shouldSwapCmpOperands(C)) {
3745 std::swap(a&: C.Op0, b&: C.Op1);
3746 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3747 }
3748
3749 adjustForTestUnderMask(DAG, DL, C);
3750 adjustICmp128(DAG, DL, C);
3751 adjustForStackGuardCompare(DAG, DL, C);
3752 return C;
3753}
3754
3755// Emit the comparison instruction described by C.
3756static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3757 if (!C.Op1.getNode()) {
3758 if (C.Opcode == SystemZISD::CMP_STACKGUARD)
3759 return DAG.getNode(Opcode: SystemZISD::CMP_STACKGUARD, DL, VT: MVT::i32, Operand: C.Op0);
3760 SDNode *Node;
3761 switch (C.Op0.getOpcode()) {
3762 case ISD::INTRINSIC_W_CHAIN:
3763 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3764 return SDValue(Node, 0);
3765 case ISD::INTRINSIC_WO_CHAIN:
3766 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3767 return SDValue(Node, Node->getNumValues() - 1);
3768 default:
3769 llvm_unreachable("Invalid comparison operands");
3770 }
3771 }
3772 if (C.Opcode == SystemZISD::ICMP)
3773 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3774 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3775 if (C.Opcode == SystemZISD::TM) {
3776 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3777 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3778 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3779 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3780 }
3781 if (C.Opcode == SystemZISD::VICMPES ||
3782 C.Opcode == SystemZISD::VICMPHS ||
3783 C.Opcode == SystemZISD::VICMPHLS ||
3784 C.Opcode == SystemZISD::VFCMPES ||
3785 C.Opcode == SystemZISD::VFCMPHS ||
3786 C.Opcode == SystemZISD::VFCMPHES) {
3787 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3788 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3789 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3790 return SDValue(Val.getNode(), 1);
3791 }
3792 if (C.Chain) {
3793 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3794 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3795 }
3796 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3797}
3798
3799// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3800// 64 bits. Extend is the extension type to use. Store the high part
3801// in Hi and the low part in Lo.
3802static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3803 SDValue Op0, SDValue Op1, SDValue &Hi,
3804 SDValue &Lo) {
3805 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3806 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3807 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3808 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3809 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3810 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3811 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3812}
3813
3814// Lower a binary operation that produces two VT results, one in each
3815// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3816// and Opcode performs the GR128 operation. Store the even register result
3817// in Even and the odd register result in Odd.
3818static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3819 unsigned Opcode, SDValue Op0, SDValue Op1,
3820 SDValue &Even, SDValue &Odd) {
3821 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
3822 bool Is32Bit = is32Bit(VT);
3823 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3824 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3825}
3826
3827// Return an i32 value that is 1 if the CC value produced by CCReg is
3828// in the mask CCMask and 0 otherwise. CC is known to have a value
3829// in CCValid, so other values can be ignored.
3830static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3831 unsigned CCValid, unsigned CCMask) {
3832 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3833 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3834 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3835 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3836 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3837}
3838
3839// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3840// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3841// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3842// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3843// floating-point comparisons.
3844enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3845static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3846 switch (CC) {
3847 case ISD::SETOEQ:
3848 case ISD::SETEQ:
3849 switch (Mode) {
3850 case CmpMode::Int: return SystemZISD::VICMPE;
3851 case CmpMode::FP: return SystemZISD::VFCMPE;
3852 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3853 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3854 }
3855 llvm_unreachable("Bad mode");
3856
3857 case ISD::SETOGE:
3858 case ISD::SETGE:
3859 switch (Mode) {
3860 case CmpMode::Int: return 0;
3861 case CmpMode::FP: return SystemZISD::VFCMPHE;
3862 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3863 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3864 }
3865 llvm_unreachable("Bad mode");
3866
3867 case ISD::SETOGT:
3868 case ISD::SETGT:
3869 switch (Mode) {
3870 case CmpMode::Int: return SystemZISD::VICMPH;
3871 case CmpMode::FP: return SystemZISD::VFCMPH;
3872 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3873 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3874 }
3875 llvm_unreachable("Bad mode");
3876
3877 case ISD::SETUGT:
3878 switch (Mode) {
3879 case CmpMode::Int: return SystemZISD::VICMPHL;
3880 case CmpMode::FP: return 0;
3881 case CmpMode::StrictFP: return 0;
3882 case CmpMode::SignalingFP: return 0;
3883 }
3884 llvm_unreachable("Bad mode");
3885
3886 default:
3887 return 0;
3888 }
3889}
3890
3891// Return the SystemZISD vector comparison operation for CC or its inverse,
3892// or 0 if neither can be done directly. Indicate in Invert whether the
3893// result is for the inverse of CC. Mode is as above.
3894static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3895 bool &Invert) {
3896 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3897 Invert = false;
3898 return Opcode;
3899 }
3900
3901 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3902 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3903 Invert = true;
3904 return Opcode;
3905 }
3906
3907 return 0;
3908}
3909
3910// Return a v2f64 that contains the extended form of elements Start and Start+1
3911// of v4f32 value Op. If Chain is nonnull, return the strict form.
3912static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3913 SDValue Op, SDValue Chain) {
3914 int Mask[] = { Start, -1, Start + 1, -1 };
3915 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3916 if (Chain) {
3917 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3918 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3919 }
3920 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3921}
3922
3923// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3924// producing a result of type VT. If Chain is nonnull, return the strict form.
3925SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3926 const SDLoc &DL, EVT VT,
3927 SDValue CmpOp0,
3928 SDValue CmpOp1,
3929 SDValue Chain) const {
3930 // There is no hardware support for v4f32 (unless we have the vector
3931 // enhancements facility 1), so extend the vector into two v2f64s
3932 // and compare those.
3933 if (CmpOp0.getValueType() == MVT::v4f32 &&
3934 !Subtarget.hasVectorEnhancements1()) {
3935 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3936 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3937 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3938 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3939 if (Chain) {
3940 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3941 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3942 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3943 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3944 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3945 H1.getValue(R: 1), L1.getValue(R: 1),
3946 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3947 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3948 SDValue Ops[2] = { Res, NewChain };
3949 return DAG.getMergeValues(Ops, dl: DL);
3950 }
3951 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3952 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3953 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3954 }
3955 if (Chain) {
3956 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3957 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3958 }
3959 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3960}
3961
3962// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3963// an integer mask of type VT. If Chain is nonnull, we have a strict
3964// floating-point comparison. If in addition IsSignaling is true, we have
3965// a strict signaling floating-point comparison.
3966SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3967 const SDLoc &DL, EVT VT,
3968 ISD::CondCode CC,
3969 SDValue CmpOp0,
3970 SDValue CmpOp1,
3971 SDValue Chain,
3972 bool IsSignaling) const {
3973 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3974 assert (!Chain || IsFP);
3975 assert (!IsSignaling || Chain);
3976 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3977 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3978 bool Invert = false;
3979 SDValue Cmp;
3980 switch (CC) {
3981 // Handle tests for order using (or (ogt y x) (oge x y)).
3982 case ISD::SETUO:
3983 Invert = true;
3984 [[fallthrough]];
3985 case ISD::SETO: {
3986 assert(IsFP && "Unexpected integer comparison");
3987 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3988 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3989 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3990 DL, VT, CmpOp0, CmpOp1, Chain);
3991 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3992 if (Chain)
3993 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3994 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3995 break;
3996 }
3997
3998 // Handle <> tests using (or (ogt y x) (ogt x y)).
3999 case ISD::SETUEQ:
4000 Invert = true;
4001 [[fallthrough]];
4002 case ISD::SETONE: {
4003 assert(IsFP && "Unexpected integer comparison");
4004 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
4005 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
4006 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
4007 DL, VT, CmpOp0, CmpOp1, Chain);
4008 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
4009 if (Chain)
4010 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4011 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
4012 break;
4013 }
4014
4015 // Otherwise a single comparison is enough. It doesn't really
4016 // matter whether we try the inversion or the swap first, since
4017 // there are no cases where both work.
4018 default:
4019 // Optimize sign-bit comparisons to signed compares.
4020 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
4021 ISD::isConstantSplatVectorAllZeros(V: CmpOp1)) {
4022 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4023 APInt Mask;
4024 if (CmpOp0.getOpcode() == ISD::AND &&
4025 ISD::isConstantSplatVector(V: CmpOp0.getOperand(i: 1), SplatValue&: Mask) &&
4026 Mask == APInt::getSignMask(BitWidth: EltSize)) {
4027 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
4028 CmpOp0 = CmpOp0.getOperand(i: 0);
4029 }
4030 }
4031 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
4032 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
4033 else {
4034 CC = ISD::getSetCCSwappedOperands(Operation: CC);
4035 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
4036 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
4037 else
4038 llvm_unreachable("Unhandled comparison");
4039 }
4040 if (Chain)
4041 Chain = Cmp.getValue(R: 1);
4042 break;
4043 }
4044 if (Invert) {
4045 SDValue Mask =
4046 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
4047 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
4048 }
4049 if (Chain && Chain.getNode() != Cmp.getNode()) {
4050 SDValue Ops[2] = { Cmp, Chain };
4051 Cmp = DAG.getMergeValues(Ops, dl: DL);
4052 }
4053 return Cmp;
4054}
4055
4056SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
4057 SelectionDAG &DAG) const {
4058 SDValue CmpOp0 = Op.getOperand(i: 0);
4059 SDValue CmpOp1 = Op.getOperand(i: 1);
4060 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
4061 SDLoc DL(Op);
4062 EVT VT = Op.getValueType();
4063 if (VT.isVector())
4064 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
4065
4066 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4067 SDValue CCReg = emitCmp(DAG, DL, C);
4068 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
4069}
4070
4071SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
4072 SelectionDAG &DAG,
4073 bool IsSignaling) const {
4074 SDValue Chain = Op.getOperand(i: 0);
4075 SDValue CmpOp0 = Op.getOperand(i: 1);
4076 SDValue CmpOp1 = Op.getOperand(i: 2);
4077 ISD::CondCode CC = Op.getOperandCondCode(i: 3);
4078 SDLoc DL(Op);
4079 EVT VT = Op.getNode()->getValueType(ResNo: 0);
4080 if (VT.isVector()) {
4081 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
4082 Chain, IsSignaling);
4083 return Res.getValue(R: Op.getResNo());
4084 }
4085
4086 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
4087 SDValue CCReg = emitCmp(DAG, DL, C);
4088 CCReg->setFlags(Op->getFlags());
4089 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
4090 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
4091 return DAG.getMergeValues(Ops, dl: DL);
4092}
4093
4094SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4095 ISD::CondCode CC = Op.getOperandCondCode(i: 1);
4096 SDValue CmpOp0 = Op.getOperand(i: 2);
4097 SDValue CmpOp1 = Op.getOperand(i: 3);
4098 SDValue Dest = Op.getOperand(i: 4);
4099 SDLoc DL(Op);
4100
4101 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4102 SDValue CCReg = emitCmp(DAG, DL, C);
4103 return DAG.getNode(
4104 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
4105 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4106 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
4107}
4108
4109// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
4110// allowing Pos and Neg to be wider than CmpOp.
4111static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
4112 return (Neg.getOpcode() == ISD::SUB &&
4113 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
4114 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
4115 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
4116 Pos.getOperand(i: 0) == CmpOp)));
4117}
4118
4119// Return the absolute or negative absolute of Op; IsNegative decides which.
4120static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
4121 bool IsNegative) {
4122 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
4123 if (IsNegative)
4124 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
4125 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
4126 return Op;
4127}
4128
4129static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
4130 Comparison C, SDValue TrueOp, SDValue FalseOp) {
4131 EVT VT = MVT::i128;
4132 unsigned Op;
4133
4134 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
4135 C.CCMask == SystemZ::CCMASK_CMP_GE ||
4136 C.CCMask == SystemZ::CCMASK_CMP_LE) {
4137 std::swap(a&: TrueOp, b&: FalseOp);
4138 C.CCMask ^= C.CCValid;
4139 }
4140 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
4141 std::swap(a&: C.Op0, b&: C.Op1);
4142 C.CCMask = SystemZ::CCMASK_CMP_GT;
4143 }
4144 switch (C.CCMask) {
4145 case SystemZ::CCMASK_CMP_EQ:
4146 Op = SystemZISD::VICMPE;
4147 break;
4148 case SystemZ::CCMASK_CMP_GT:
4149 if (C.ICmpType == SystemZICMP::UnsignedOnly)
4150 Op = SystemZISD::VICMPHL;
4151 else
4152 Op = SystemZISD::VICMPH;
4153 break;
4154 default:
4155 llvm_unreachable("Unhandled comparison");
4156 break;
4157 }
4158
4159 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
4160 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
4161 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
4162 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
4163}
4164
4165SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4166 SelectionDAG &DAG) const {
4167 SDValue CmpOp0 = Op.getOperand(i: 0);
4168 SDValue CmpOp1 = Op.getOperand(i: 1);
4169 SDValue TrueOp = Op.getOperand(i: 2);
4170 SDValue FalseOp = Op.getOperand(i: 3);
4171 ISD::CondCode CC = Op.getOperandCondCode(i: 4);
4172 SDLoc DL(Op);
4173
4174 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4175 // legalizer, as it will be handled according to the type of the resulting
4176 // value. Extend them here if needed.
4177 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4178 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
4179 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
4180 }
4181
4182 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4183
4184 // Check for absolute and negative-absolute selections, including those
4185 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4186 // This check supplements the one in DAGCombiner.
4187 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4188 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4189 C.Op1.getOpcode() == ISD::Constant &&
4190 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
4191 C.Op1->getAsZExtVal() == 0) {
4192 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
4193 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
4194 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
4195 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
4196 }
4197
4198 if (Subtarget.hasVectorEnhancements3() &&
4199 C.Opcode == SystemZISD::ICMP &&
4200 C.Op0.getValueType() == MVT::i128 &&
4201 TrueOp.getValueType() == MVT::i128) {
4202 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4203 }
4204
4205 SDValue CCReg = emitCmp(DAG, DL, C);
4206 SDValue Ops[] = {TrueOp, FalseOp,
4207 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4208 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
4209
4210 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
4211}
4212
4213SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4214 SelectionDAG &DAG) const {
4215 SDLoc DL(Node);
4216 const GlobalValue *GV = Node->getGlobal();
4217 int64_t Offset = Node->getOffset();
4218 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4219 CodeModel::Model CM = DAG.getTarget().getCodeModel();
4220
4221 SDValue Result;
4222 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4223 if (isInt<32>(x: Offset)) {
4224 // Assign anchors at 1<<12 byte boundaries.
4225 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4226 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
4227 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4228
4229 // The offset can be folded into the address if it is aligned to a
4230 // halfword.
4231 Offset -= Anchor;
4232 if (Offset != 0 && (Offset & 1) == 0) {
4233 SDValue Full =
4234 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
4235 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
4236 Offset = 0;
4237 }
4238 } else {
4239 // Conservatively load a constant offset greater than 32 bits into a
4240 // register below.
4241 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
4242 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4243 }
4244 } else if (Subtarget.isTargetELF()) {
4245 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
4246 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4247 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
4248 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4249 } else if (Subtarget.isTargetzOS()) {
4250 Result = getADAEntry(DAG, GV, DL, PtrVT);
4251 } else
4252 llvm_unreachable("Unexpected Subtarget");
4253
4254 // If there was a non-zero offset that we didn't fold, create an explicit
4255 // addition for it.
4256 if (Offset != 0)
4257 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4258 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4259
4260 return Result;
4261}
4262
4263SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4264 SelectionDAG &DAG,
4265 unsigned Opcode,
4266 SDValue GOTOffset) const {
4267 SDLoc DL(Node);
4268 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4269 SDValue Chain = DAG.getEntryNode();
4270 SDValue Glue;
4271
4272 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4273 CallingConv::GHC)
4274 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4275
4276 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4277 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4278 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4279 Glue = Chain.getValue(R: 1);
4280 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4281 Glue = Chain.getValue(R: 1);
4282
4283 // The first call operand is the chain and the second is the TLS symbol.
4284 SmallVector<SDValue, 8> Ops;
4285 Ops.push_back(Elt: Chain);
4286 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4287 VT: Node->getValueType(ResNo: 0),
4288 offset: 0, TargetFlags: 0));
4289
4290 // Add argument registers to the end of the list so that they are
4291 // known live into the call.
4292 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4293 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4294
4295 // Add a register mask operand representing the call-preserved registers.
4296 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4297 const uint32_t *Mask =
4298 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4299 assert(Mask && "Missing call preserved mask for calling convention");
4300 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4301
4302 // Glue the call to the argument copies.
4303 Ops.push_back(Elt: Glue);
4304
4305 // Emit the call.
4306 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4307 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4308 Glue = Chain.getValue(R: 1);
4309
4310 // Copy the return value from %r2.
4311 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4312}
4313
4314SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4315 SelectionDAG &DAG) const {
4316 SDValue Chain = DAG.getEntryNode();
4317 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4318
4319 // The high part of the thread pointer is in access register 0.
4320 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4321 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4322
4323 // The low part of the thread pointer is in access register 1.
4324 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4325 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4326
4327 // Merge them into a single 64-bit address.
4328 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4329 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4330 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4331}
4332
4333SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4334 SelectionDAG &DAG) const {
4335 if (DAG.getTarget().useEmulatedTLS())
4336 return LowerToTLSEmulatedModel(GA: Node, DAG);
4337 SDLoc DL(Node);
4338 const GlobalValue *GV = Node->getGlobal();
4339 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4340 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4341
4342 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4343 CallingConv::GHC)
4344 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4345
4346 SDValue TP = lowerThreadPointer(DL, DAG);
4347
4348 // Get the offset of GA from the thread pointer, based on the TLS model.
4349 SDValue Offset;
4350 switch (model) {
4351 case TLSModel::GeneralDynamic: {
4352 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4353 SystemZConstantPoolValue *CPV =
4354 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4355
4356 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4357 Offset = DAG.getLoad(
4358 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4359 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4360
4361 // Call __tls_get_offset to retrieve the offset.
4362 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4363 break;
4364 }
4365
4366 case TLSModel::LocalDynamic: {
4367 // Load the GOT offset of the module ID.
4368 SystemZConstantPoolValue *CPV =
4369 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4370
4371 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4372 Offset = DAG.getLoad(
4373 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4374 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4375
4376 // Call __tls_get_offset to retrieve the module base offset.
4377 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4378
4379 // Note: The SystemZLDCleanupPass will remove redundant computations
4380 // of the module base offset. Count total number of local-dynamic
4381 // accesses to trigger execution of that pass.
4382 SystemZMachineFunctionInfo* MFI =
4383 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4384 MFI->incNumLocalDynamicTLSAccesses();
4385
4386 // Add the per-symbol offset.
4387 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4388
4389 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4390 DTPOffset = DAG.getLoad(
4391 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4392 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4393
4394 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4395 break;
4396 }
4397
4398 case TLSModel::InitialExec: {
4399 // Load the offset from the GOT.
4400 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4401 TargetFlags: SystemZII::MO_INDNTPOFF);
4402 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4403 Offset =
4404 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4405 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4406 break;
4407 }
4408
4409 case TLSModel::LocalExec: {
4410 // Force the offset into the constant pool and load it from there.
4411 SystemZConstantPoolValue *CPV =
4412 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4413
4414 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4415 Offset = DAG.getLoad(
4416 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4417 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4418 break;
4419 }
4420 }
4421
4422 // Add the base and offset together.
4423 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4424}
4425
4426SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4427 SelectionDAG &DAG) const {
4428 SDLoc DL(Node);
4429 const BlockAddress *BA = Node->getBlockAddress();
4430 int64_t Offset = Node->getOffset();
4431 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4432
4433 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4434 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4435 return Result;
4436}
4437
4438SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4439 SelectionDAG &DAG) const {
4440 SDLoc DL(JT);
4441 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4442 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4443
4444 // Use LARL to load the address of the table.
4445 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4446}
4447
4448SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4449 SelectionDAG &DAG) const {
4450 SDLoc DL(CP);
4451 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4452
4453 SDValue Result;
4454 if (CP->isMachineConstantPoolEntry())
4455 Result =
4456 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4457 else
4458 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4459 Offset: CP->getOffset());
4460
4461 // Use LARL to load the address of the constant pool entry.
4462 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4463}
4464
4465SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4466 SelectionDAG &DAG) const {
4467 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4468 MachineFunction &MF = DAG.getMachineFunction();
4469 MachineFrameInfo &MFI = MF.getFrameInfo();
4470 MFI.setFrameAddressIsTaken(true);
4471
4472 SDLoc DL(Op);
4473 unsigned Depth = Op.getConstantOperandVal(i: 0);
4474 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4475
4476 // By definition, the frame address is the address of the back chain. (In
4477 // the case of packed stack without backchain, return the address where the
4478 // backchain would have been stored. This will either be an unused space or
4479 // contain a saved register).
4480 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4481 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4482
4483 if (Depth > 0) {
4484 // FIXME The frontend should detect this case.
4485 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4486 report_fatal_error(reason: "Unsupported stack frame traversal count");
4487
4488 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4489 while (Depth--) {
4490 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4491 PtrInfo: MachinePointerInfo());
4492 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4493 }
4494 }
4495
4496 return BackChain;
4497}
4498
4499SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4500 SelectionDAG &DAG) const {
4501 MachineFunction &MF = DAG.getMachineFunction();
4502 MachineFrameInfo &MFI = MF.getFrameInfo();
4503 MFI.setReturnAddressIsTaken(true);
4504
4505 SDLoc DL(Op);
4506 unsigned Depth = Op.getConstantOperandVal(i: 0);
4507 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4508
4509 if (Depth > 0) {
4510 // FIXME The frontend should detect this case.
4511 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4512 report_fatal_error(reason: "Unsupported stack frame traversal count");
4513
4514 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4515 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4516 int Offset = TFL->getReturnAddressOffset(MF);
4517 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4518 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4519 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4520 PtrInfo: MachinePointerInfo());
4521 }
4522
4523 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4524 // implicit live-in.
4525 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4526 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4527 RC: &SystemZ::GR64BitRegClass);
4528 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4529}
4530
4531SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4532 SelectionDAG &DAG) const {
4533 SDLoc DL(Op);
4534 SDValue In = Op.getOperand(i: 0);
4535 EVT InVT = In.getValueType();
4536 EVT ResVT = Op.getValueType();
4537
4538 // Convert loads directly. This is normally done by DAGCombiner,
4539 // but we need this case for bitcasts that are created during lowering
4540 // and which are then lowered themselves.
4541 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4542 if (ISD::isNormalLoad(N: LoadN)) {
4543 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4544 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4545 // Update the chain uses.
4546 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4547 return NewLoad;
4548 }
4549
4550 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4551 SDValue In64;
4552 if (Subtarget.hasHighWord()) {
4553 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4554 VT: MVT::i64);
4555 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4556 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4557 } else {
4558 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4559 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4560 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4561 }
4562 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4563 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4564 DL, VT: MVT::f32, Operand: Out64);
4565 }
4566 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4567 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4568 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4569 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4570 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4571 if (Subtarget.hasHighWord())
4572 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4573 VT: MVT::i32, Operand: Out64);
4574 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4575 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4576 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4577 }
4578 llvm_unreachable("Unexpected bitcast combination");
4579}
4580
4581SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4582 SelectionDAG &DAG) const {
4583
4584 if (Subtarget.isTargetXPLINK64())
4585 return lowerVASTART_XPLINK(Op, DAG);
4586 else
4587 return lowerVASTART_ELF(Op, DAG);
4588}
4589
4590SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4591 SelectionDAG &DAG) const {
4592 MachineFunction &MF = DAG.getMachineFunction();
4593 SystemZMachineFunctionInfo *FuncInfo =
4594 MF.getInfo<SystemZMachineFunctionInfo>();
4595
4596 SDLoc DL(Op);
4597
4598 // vastart just stores the address of the VarArgsFrameIndex slot into the
4599 // memory location argument.
4600 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4601 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4602 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4603 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4604 PtrInfo: MachinePointerInfo(SV));
4605}
4606
4607SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4608 SelectionDAG &DAG) const {
4609 MachineFunction &MF = DAG.getMachineFunction();
4610 SystemZMachineFunctionInfo *FuncInfo =
4611 MF.getInfo<SystemZMachineFunctionInfo>();
4612 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4613
4614 SDValue Chain = Op.getOperand(i: 0);
4615 SDValue Addr = Op.getOperand(i: 1);
4616 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4617 SDLoc DL(Op);
4618
4619 // The initial values of each field.
4620 const unsigned NumFields = 4;
4621 SDValue Fields[NumFields] = {
4622 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4623 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4624 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4625 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4626 };
4627
4628 // Store each field into its respective slot.
4629 SDValue MemOps[NumFields];
4630 unsigned Offset = 0;
4631 for (unsigned I = 0; I < NumFields; ++I) {
4632 SDValue FieldAddr = Addr;
4633 if (Offset != 0)
4634 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4635 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4636 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4637 PtrInfo: MachinePointerInfo(SV, Offset));
4638 Offset += 8;
4639 }
4640 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4641}
4642
4643SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4644 SelectionDAG &DAG) const {
4645 SDValue Chain = Op.getOperand(i: 0);
4646 SDValue DstPtr = Op.getOperand(i: 1);
4647 SDValue SrcPtr = Op.getOperand(i: 2);
4648 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4649 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4650 SDLoc DL(Op);
4651
4652 uint32_t Sz =
4653 Subtarget.isTargetXPLINK64() ? DAG.getDataLayout().getPointerSize(AS: 0) : 32;
4654 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4655 DstAlign: Align(8), SrcAlign: Align(8), /*isVolatile*/ isVol: false,
4656 /*AlwaysInline*/ false,
4657 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4658 SrcPtrInfo: MachinePointerInfo(SrcSV));
4659}
4660
4661SDValue
4662SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4663 SelectionDAG &DAG) const {
4664 if (Subtarget.isTargetXPLINK64())
4665 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4666 else
4667 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4668}
4669
4670SDValue
4671SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4672 SelectionDAG &DAG) const {
4673 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4674 MachineFunction &MF = DAG.getMachineFunction();
4675 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4676 SDValue Chain = Op.getOperand(i: 0);
4677 SDValue Size = Op.getOperand(i: 1);
4678 SDValue Align = Op.getOperand(i: 2);
4679 SDLoc DL(Op);
4680
4681 // If user has set the no alignment function attribute, ignore
4682 // alloca alignments.
4683 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4684
4685 uint64_t StackAlign = TFI->getStackAlignment();
4686 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4687 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4688
4689 SDValue NeededSpace = Size;
4690
4691 // Add extra space for alignment if needed.
4692 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4693 if (ExtraAlignSpace)
4694 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4695 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4696
4697 bool IsSigned = false;
4698 bool DoesNotReturn = false;
4699 bool IsReturnValueUsed = false;
4700 EVT VT = Op.getValueType();
4701 SDValue AllocaCall =
4702 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4703 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4704 IsReturnValueUsed)
4705 .first;
4706
4707 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4708 // to end of call in order to ensure it isn't broken up from the call
4709 // sequence.
4710 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4711 Register SPReg = Regs.getStackPointerRegister();
4712 Chain = AllocaCall.getValue(R: 1);
4713 SDValue Glue = AllocaCall.getValue(R: 2);
4714 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4715 Chain = NewSPRegNode.getValue(R: 1);
4716
4717 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4718 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4719 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4720
4721 // Dynamically realign if needed.
4722 if (ExtraAlignSpace) {
4723 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4724 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4725 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4726 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4727 }
4728
4729 SDValue Ops[2] = {Result, Chain};
4730 return DAG.getMergeValues(Ops, dl: DL);
4731}
4732
4733SDValue
4734SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4735 SelectionDAG &DAG) const {
4736 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4737 MachineFunction &MF = DAG.getMachineFunction();
4738 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4739 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4740
4741 SDValue Chain = Op.getOperand(i: 0);
4742 SDValue Size = Op.getOperand(i: 1);
4743 SDValue Align = Op.getOperand(i: 2);
4744 SDLoc DL(Op);
4745
4746 // If user has set the no alignment function attribute, ignore
4747 // alloca alignments.
4748 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4749
4750 uint64_t StackAlign = TFI->getStackAlignment();
4751 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4752 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4753
4754 Register SPReg = getStackPointerRegisterToSaveRestore();
4755 SDValue NeededSpace = Size;
4756
4757 // Get a reference to the stack pointer.
4758 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4759
4760 // If we need a backchain, save it now.
4761 SDValue Backchain;
4762 if (StoreBackchain)
4763 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4764 PtrInfo: MachinePointerInfo());
4765
4766 // Add extra space for alignment if needed.
4767 if (ExtraAlignSpace)
4768 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4769 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4770
4771 // Get the new stack pointer value.
4772 SDValue NewSP;
4773 if (hasInlineStackProbe(MF)) {
4774 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4775 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4776 Chain = NewSP.getValue(R: 1);
4777 }
4778 else {
4779 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4780 // Copy the new stack pointer back.
4781 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4782 }
4783
4784 // The allocated data lives above the 160 bytes allocated for the standard
4785 // frame, plus any outgoing stack arguments. We don't know how much that
4786 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4787 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4788 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4789
4790 // Dynamically realign if needed.
4791 if (RequiredAlign > StackAlign) {
4792 Result =
4793 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4794 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4795 Result =
4796 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4797 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4798 }
4799
4800 if (StoreBackchain)
4801 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4802 PtrInfo: MachinePointerInfo());
4803
4804 SDValue Ops[2] = { Result, Chain };
4805 return DAG.getMergeValues(Ops, dl: DL);
4806}
4807
4808SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4809 SDValue Op, SelectionDAG &DAG) const {
4810 SDLoc DL(Op);
4811
4812 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4813}
4814
4815SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4816 SelectionDAG &DAG,
4817 unsigned Opcode) const {
4818 EVT VT = Op.getValueType();
4819 SDLoc DL(Op);
4820 SDValue Even, Odd;
4821
4822 // This custom expander is only used on z17 and later for 64-bit types.
4823 assert(!is32Bit(VT));
4824 assert(Subtarget.hasMiscellaneousExtensions2());
4825
4826 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4827 // the high result in the even register. Return the latter.
4828 lowerGR128Binary(DAG, DL, VT, Opcode,
4829 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4830 return Even;
4831}
4832
4833SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4834 SelectionDAG &DAG) const {
4835 EVT VT = Op.getValueType();
4836 SDLoc DL(Op);
4837 SDValue Ops[2];
4838 if (is32Bit(VT))
4839 // Just do a normal 64-bit multiplication and extract the results.
4840 // We define this so that it can be used for constant division.
4841 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4842 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4843 else if (Subtarget.hasMiscellaneousExtensions2())
4844 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4845 // the high result in the even register. ISD::SMUL_LOHI is defined to
4846 // return the low half first, so the results are in reverse order.
4847 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4848 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4849 else {
4850 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4851 //
4852 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4853 //
4854 // but using the fact that the upper halves are either all zeros
4855 // or all ones:
4856 //
4857 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4858 //
4859 // and grouping the right terms together since they are quicker than the
4860 // multiplication:
4861 //
4862 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4863 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4864 SDValue LL = Op.getOperand(i: 0);
4865 SDValue RL = Op.getOperand(i: 1);
4866 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4867 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4868 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4869 // the high result in the even register. ISD::SMUL_LOHI is defined to
4870 // return the low half first, so the results are in reverse order.
4871 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4872 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4873 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4874 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4875 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4876 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4877 }
4878 return DAG.getMergeValues(Ops, dl: DL);
4879}
4880
4881SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4882 SelectionDAG &DAG) const {
4883 EVT VT = Op.getValueType();
4884 SDLoc DL(Op);
4885 SDValue Ops[2];
4886 if (is32Bit(VT))
4887 // Just do a normal 64-bit multiplication and extract the results.
4888 // We define this so that it can be used for constant division.
4889 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4890 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4891 else
4892 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4893 // the high result in the even register. ISD::UMUL_LOHI is defined to
4894 // return the low half first, so the results are in reverse order.
4895 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4896 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4897 return DAG.getMergeValues(Ops, dl: DL);
4898}
4899
4900SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4901 SelectionDAG &DAG) const {
4902 SDValue Op0 = Op.getOperand(i: 0);
4903 SDValue Op1 = Op.getOperand(i: 1);
4904 EVT VT = Op.getValueType();
4905 SDLoc DL(Op);
4906
4907 // We use DSGF for 32-bit division. This means the first operand must
4908 // always be 64-bit, and the second operand should be 32-bit whenever
4909 // that is possible, to improve performance.
4910 if (is32Bit(VT))
4911 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4912 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4913 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4914
4915 // DSG(F) returns the remainder in the even register and the
4916 // quotient in the odd register.
4917 SDValue Ops[2];
4918 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4919 return DAG.getMergeValues(Ops, dl: DL);
4920}
4921
4922SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4923 SelectionDAG &DAG) const {
4924 EVT VT = Op.getValueType();
4925 SDLoc DL(Op);
4926
4927 // DL(G) returns the remainder in the even register and the
4928 // quotient in the odd register.
4929 SDValue Ops[2];
4930 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4931 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4932 return DAG.getMergeValues(Ops, dl: DL);
4933}
4934
4935SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4936 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4937
4938 // Get the known-zero masks for each operand.
4939 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4940 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4941 DAG.computeKnownBits(Op: Ops[1])};
4942
4943 // See if the upper 32 bits of one operand and the lower 32 bits of the
4944 // other are known zero. They are the low and high operands respectively.
4945 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4946 Known[1].Zero.getZExtValue() };
4947 unsigned High, Low;
4948 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4949 High = 1, Low = 0;
4950 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4951 High = 0, Low = 1;
4952 else
4953 return Op;
4954
4955 SDValue LowOp = Ops[Low];
4956 SDValue HighOp = Ops[High];
4957
4958 // If the high part is a constant, we're better off using IILH.
4959 if (HighOp.getOpcode() == ISD::Constant)
4960 return Op;
4961
4962 // If the low part is a constant that is outside the range of LHI,
4963 // then we're better off using IILF.
4964 if (LowOp.getOpcode() == ISD::Constant) {
4965 int64_t Value = int32_t(LowOp->getAsZExtVal());
4966 if (!isInt<16>(x: Value))
4967 return Op;
4968 }
4969
4970 // Check whether the high part is an AND that doesn't change the
4971 // high 32 bits and just masks out low bits. We can skip it if so.
4972 if (HighOp.getOpcode() == ISD::AND &&
4973 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4974 SDValue HighOp0 = HighOp.getOperand(i: 0);
4975 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4976 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4977 HighOp = HighOp0;
4978 }
4979
4980 // Take advantage of the fact that all GR32 operations only change the
4981 // low 32 bits by truncating Low to an i32 and inserting it directly
4982 // using a subreg. The interesting cases are those where the truncation
4983 // can be folded.
4984 SDLoc DL(Op);
4985 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4986 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4987 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4988}
4989
4990// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4991SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4992 SelectionDAG &DAG) const {
4993 SDNode *N = Op.getNode();
4994 SDValue LHS = N->getOperand(Num: 0);
4995 SDValue RHS = N->getOperand(Num: 1);
4996 SDLoc DL(N);
4997
4998 if (N->getValueType(ResNo: 0) == MVT::i128) {
4999 unsigned BaseOp = 0;
5000 unsigned FlagOp = 0;
5001 bool IsBorrow = false;
5002 switch (Op.getOpcode()) {
5003 default: llvm_unreachable("Unknown instruction!");
5004 case ISD::UADDO:
5005 BaseOp = ISD::ADD;
5006 FlagOp = SystemZISD::VACC;
5007 break;
5008 case ISD::USUBO:
5009 BaseOp = ISD::SUB;
5010 FlagOp = SystemZISD::VSCBI;
5011 IsBorrow = true;
5012 break;
5013 }
5014 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
5015 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
5016 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
5017 N2: DAG.getValueType(MVT::i1));
5018 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
5019 if (IsBorrow)
5020 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
5021 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
5022 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
5023 }
5024
5025 unsigned BaseOp = 0;
5026 unsigned CCValid = 0;
5027 unsigned CCMask = 0;
5028
5029 switch (Op.getOpcode()) {
5030 default: llvm_unreachable("Unknown instruction!");
5031 case ISD::SADDO:
5032 BaseOp = SystemZISD::SADDO;
5033 CCValid = SystemZ::CCMASK_ARITH;
5034 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
5035 break;
5036 case ISD::SSUBO:
5037 BaseOp = SystemZISD::SSUBO;
5038 CCValid = SystemZ::CCMASK_ARITH;
5039 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
5040 break;
5041 case ISD::UADDO:
5042 BaseOp = SystemZISD::UADDO;
5043 CCValid = SystemZ::CCMASK_LOGICAL;
5044 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
5045 break;
5046 case ISD::USUBO:
5047 BaseOp = SystemZISD::USUBO;
5048 CCValid = SystemZ::CCMASK_LOGICAL;
5049 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
5050 break;
5051 }
5052
5053 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
5054 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
5055
5056 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
5057 if (N->getValueType(ResNo: 1) == MVT::i1)
5058 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5059
5060 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5061}
5062
5063static bool isAddCarryChain(SDValue Carry) {
5064 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
5065 Carry->getValueType(ResNo: 0) != MVT::i128)
5066 Carry = Carry.getOperand(i: 2);
5067 return Carry.getOpcode() == ISD::UADDO &&
5068 Carry->getValueType(ResNo: 0) != MVT::i128;
5069}
5070
5071static bool isSubBorrowChain(SDValue Carry) {
5072 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
5073 Carry->getValueType(ResNo: 0) != MVT::i128)
5074 Carry = Carry.getOperand(i: 2);
5075 return Carry.getOpcode() == ISD::USUBO &&
5076 Carry->getValueType(ResNo: 0) != MVT::i128;
5077}
5078
5079// Lower UADDO_CARRY/USUBO_CARRY nodes.
5080SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
5081 SelectionDAG &DAG) const {
5082
5083 SDNode *N = Op.getNode();
5084 MVT VT = N->getSimpleValueType(ResNo: 0);
5085
5086 // Let legalize expand this if it isn't a legal type yet.
5087 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
5088 return SDValue();
5089
5090 SDValue LHS = N->getOperand(Num: 0);
5091 SDValue RHS = N->getOperand(Num: 1);
5092 SDValue Carry = Op.getOperand(i: 2);
5093 SDLoc DL(N);
5094
5095 if (VT == MVT::i128) {
5096 unsigned BaseOp = 0;
5097 unsigned FlagOp = 0;
5098 bool IsBorrow = false;
5099 switch (Op.getOpcode()) {
5100 default: llvm_unreachable("Unknown instruction!");
5101 case ISD::UADDO_CARRY:
5102 BaseOp = SystemZISD::VAC;
5103 FlagOp = SystemZISD::VACCC;
5104 break;
5105 case ISD::USUBO_CARRY:
5106 BaseOp = SystemZISD::VSBI;
5107 FlagOp = SystemZISD::VSBCBI;
5108 IsBorrow = true;
5109 break;
5110 }
5111 if (IsBorrow)
5112 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
5113 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
5114 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
5115 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5116 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5117 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
5118 N2: DAG.getValueType(MVT::i1));
5119 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
5120 if (IsBorrow)
5121 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
5122 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
5123 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
5124 }
5125
5126 unsigned BaseOp = 0;
5127 unsigned CCValid = 0;
5128 unsigned CCMask = 0;
5129
5130 switch (Op.getOpcode()) {
5131 default: llvm_unreachable("Unknown instruction!");
5132 case ISD::UADDO_CARRY:
5133 if (!isAddCarryChain(Carry))
5134 return SDValue();
5135
5136 BaseOp = SystemZISD::ADDCARRY;
5137 CCValid = SystemZ::CCMASK_LOGICAL;
5138 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
5139 break;
5140 case ISD::USUBO_CARRY:
5141 if (!isSubBorrowChain(Carry))
5142 return SDValue();
5143
5144 BaseOp = SystemZISD::SUBCARRY;
5145 CCValid = SystemZ::CCMASK_LOGICAL;
5146 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
5147 break;
5148 }
5149
5150 // Set the condition code from the carry flag.
5151 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
5152 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
5153 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
5154
5155 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
5156 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
5157
5158 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
5159 if (N->getValueType(ResNo: 1) == MVT::i1)
5160 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5161
5162 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5163}
5164
5165SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5166 SelectionDAG &DAG) const {
5167 EVT VT = Op.getValueType();
5168 SDLoc DL(Op);
5169 Op = Op.getOperand(i: 0);
5170
5171 if (VT.getScalarSizeInBits() == 128) {
5172 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
5173 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
5174 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
5175 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
5176 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5177 return Op;
5178 }
5179
5180 // Handle vector types via VPOPCT.
5181 if (VT.isVector()) {
5182 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
5183 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
5184 switch (VT.getScalarSizeInBits()) {
5185 case 8:
5186 break;
5187 case 16: {
5188 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
5189 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
5190 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5191 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5192 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5193 break;
5194 }
5195 case 32: {
5196 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5197 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5198 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5199 break;
5200 }
5201 case 64: {
5202 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5203 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5204 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
5205 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v4i32, Operand: Tmp);
5206 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5207 break;
5208 }
5209 default:
5210 llvm_unreachable("Unexpected type");
5211 }
5212 return Op;
5213 }
5214
5215 // Get the known-zero mask for the operand.
5216 KnownBits Known = DAG.computeKnownBits(Op);
5217 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5218 if (NumSignificantBits == 0)
5219 return DAG.getConstant(Val: 0, DL, VT);
5220
5221 // Skip known-zero high parts of the operand.
5222 int64_t OrigBitSize = VT.getSizeInBits();
5223 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
5224 BitSize = std::min(a: BitSize, b: OrigBitSize);
5225
5226 // The POPCNT instruction counts the number of bits in each byte.
5227 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
5228 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
5229 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
5230
5231 // Add up per-byte counts in a binary tree. All bits of Op at
5232 // position larger than BitSize remain zero throughout.
5233 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5234 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
5235 if (BitSize != OrigBitSize)
5236 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
5237 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
5238 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5239 }
5240
5241 // Extract overall result from high byte.
5242 if (BitSize > 8)
5243 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
5244 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
5245
5246 return Op;
5247}
5248
5249SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5250 SelectionDAG &DAG) const {
5251 SDLoc DL(Op);
5252 AtomicOrdering FenceOrdering =
5253 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5254 SyncScope::ID FenceSSID =
5255 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5256
5257 // The only fence that needs an instruction is a sequentially-consistent
5258 // cross-thread fence.
5259 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5260 FenceSSID == SyncScope::System) {
5261 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5262 Op1: Op.getOperand(i: 0)),
5263 0);
5264 }
5265
5266 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5267 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5268}
5269
5270SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5271 SelectionDAG &DAG) const {
5272 EVT RegVT = Op.getValueType();
5273 if (RegVT.getSizeInBits() == 128)
5274 return lowerATOMIC_LDST_I128(Op, DAG);
5275 return lowerLoadF16(Op, DAG);
5276}
5277
5278SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5279 SelectionDAG &DAG) const {
5280 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5281 if (Node->getMemoryVT().getSizeInBits() == 128)
5282 return lowerATOMIC_LDST_I128(Op, DAG);
5283 return lowerStoreF16(Op, DAG);
5284}
5285
5286SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5287 SelectionDAG &DAG) const {
5288 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5289 assert(
5290 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5291 "Only custom lowering i128 or f128.");
5292 // Use same code to handle both legal and non-legal i128 types.
5293 SmallVector<SDValue, 2> Results;
5294 LowerOperationWrapper(N: Node, Results, DAG);
5295 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5296}
5297
5298// Prepare for a Compare And Swap for a subword operation. This needs to be
5299// done in memory with 4 bytes at natural alignment.
5300static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5301 SDValue &AlignedAddr, SDValue &BitShift,
5302 SDValue &NegBitShift) {
5303 EVT PtrVT = Addr.getValueType();
5304 EVT WideVT = MVT::i32;
5305
5306 // Get the address of the containing word.
5307 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5308 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5309
5310 // Get the number of bits that the word must be rotated left in order
5311 // to bring the field to the top bits of a GR32.
5312 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5313 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5314 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5315
5316 // Get the complementing shift amount, for rotating a field in the top
5317 // bits back to its proper position.
5318 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5319 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5320
5321}
5322
5323// N is an 8- or 16-bit ATOMIC_LOAD_* operation. Lower it into the fullword
5324// ATOMIC_LOADW_* operation.
5325void SystemZTargetLowering::lowerATOMIC_LOAD_OP(
5326 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,
5327 unsigned Opcode) const {
5328 auto *Node = cast<AtomicSDNode>(Val: N);
5329
5330 EVT NarrowVT = Node->getMemoryVT();
5331 EVT WideVT = MVT::i32;
5332 int64_t BitSize = NarrowVT.getSizeInBits();
5333 SDValue ChainIn = Node->getChain();
5334 SDValue Addr = Node->getBasePtr();
5335 SDValue Src2 = Node->getVal();
5336 MachineMemOperand *MMO = Node->getMemOperand();
5337 SDLoc DL(Node);
5338
5339 // Convert atomic subtracts of constants into additions.
5340 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5341 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5342 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5343 Src2 = DAG.getConstant(Val: -Const->getAPIntValue(), DL, VT: Src2.getValueType());
5344 }
5345
5346 SDValue AlignedAddr, BitShift, NegBitShift;
5347 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5348
5349 // Extend the source operand to 32 bits and prepare it for the inner loop.
5350 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5351 // operations require the source to be shifted in advance. (This shift
5352 // can be folded if the source is constant.) For AND and NAND, the lower
5353 // bits must be set, while for other opcodes they should be left clear.
5354 Src2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WideVT, Operand: Src2);
5355 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5356 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5357 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5358 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5359 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5360 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5361 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5362
5363 // Construct the ATOMIC_LOADW_* node.
5364 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5365 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5366 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5367 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5368 MemVT: NarrowVT, MMO);
5369
5370 // Rotate the result of the final CS so that the field is in the lower
5371 // bits of a GR32, then truncate it.
5372 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5373 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5374 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5375 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: NarrowVT, Operand: Result));
5376 Results.push_back(Elt: AtomicOp.getValue(R: 1));
5377}
5378
5379// Op is a 32- or 64-bit ATOMIC_LOAD_SUB operation. Convert it into an
5380// addition.
5381SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5382 SelectionDAG &DAG) const {
5383 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5384 EVT MemVT = Node->getMemoryVT();
5385 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5386 assert(Subtarget.hasInterlockedAccess1() &&
5387 "Should have been expanded by AtomicExpand pass.");
5388 SDValue Src2 = Node->getVal();
5389 SDLoc DL(Src2);
5390 SDValue NegSrc2 =
5391 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5392 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT, Chain: Node->getChain(),
5393 Ptr: Node->getBasePtr(), Val: NegSrc2, MMO: Node->getMemOperand());
5394}
5395
5396// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5397SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5398 SelectionDAG &DAG) const {
5399 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5400 SDValue ChainIn = Node->getOperand(Num: 0);
5401 SDValue Addr = Node->getOperand(Num: 1);
5402 SDValue CmpVal = Node->getOperand(Num: 2);
5403 SDValue SwapVal = Node->getOperand(Num: 3);
5404 MachineMemOperand *MMO = Node->getMemOperand();
5405 SDLoc DL(Node);
5406
5407 if (Node->getMemoryVT() == MVT::i128) {
5408 // Use same code to handle both legal and non-legal i128 types.
5409 SmallVector<SDValue, 3> Results;
5410 LowerOperationWrapper(N: Node, Results, DAG);
5411 return DAG.getMergeValues(Ops: Results, dl: DL);
5412 }
5413
5414 // We have native support for 32-bit and 64-bit compare and swap, but we
5415 // still need to expand extracting the "success" result from the CC.
5416 EVT NarrowVT = Node->getMemoryVT();
5417 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5418 if (NarrowVT == WideVT) {
5419 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5420 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5421 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5422 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5423 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5424 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5425
5426 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5427 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5428 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5429 return SDValue();
5430 }
5431
5432 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5433 // via a fullword ATOMIC_CMP_SWAPW operation.
5434 int64_t BitSize = NarrowVT.getSizeInBits();
5435
5436 SDValue AlignedAddr, BitShift, NegBitShift;
5437 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5438
5439 // Construct the ATOMIC_CMP_SWAPW node.
5440 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5441 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5442 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5443 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5444 VTList, Ops, MemVT: NarrowVT, MMO);
5445 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5446 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5447
5448 // emitAtomicCmpSwapW() will zero extend the result (original value).
5449 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5450 N2: DAG.getValueType(NarrowVT));
5451 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5452 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5453 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5454 return SDValue();
5455}
5456
5457MachineMemOperand::Flags
5458SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5459 // Because of how we convert atomic_load and atomic_store to normal loads and
5460 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5461 // since DAGCombine hasn't been updated to account for atomic, but non
5462 // volatile loads. (See D57601)
5463 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5464 if (SI->isAtomic())
5465 return MachineMemOperand::MOVolatile;
5466 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5467 if (LI->isAtomic())
5468 return MachineMemOperand::MOVolatile;
5469 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5470 if (AI->isAtomic())
5471 return MachineMemOperand::MOVolatile;
5472 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5473 if (AI->isAtomic())
5474 return MachineMemOperand::MOVolatile;
5475 return MachineMemOperand::MONone;
5476}
5477
5478SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5479 SelectionDAG &DAG) const {
5480 MachineFunction &MF = DAG.getMachineFunction();
5481 auto *Regs = Subtarget.getSpecialRegisters();
5482 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5483 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5484 "in GHC calling convention");
5485 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5486 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5487}
5488
5489SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5490 SelectionDAG &DAG) const {
5491 MachineFunction &MF = DAG.getMachineFunction();
5492 auto *Regs = Subtarget.getSpecialRegisters();
5493
5494 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5495 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5496 "in GHC calling convention");
5497
5498 SDValue Chain = Op.getOperand(i: 0);
5499 SDValue NewSP = Op.getOperand(i: 1);
5500 SDLoc DL(Op);
5501
5502 // XPLINK: @@ALCAXP (lowerDYNAMIC_STACKALLOC_XPLINK) moves the frame header
5503 // (register save area and reserved words at SP + bias) down with the stack
5504 // pointer, and calls made while the stack pointer is lowered place their
5505 // arguments over the old header. The current header must be copied back to
5506 // the restored stack pointer, otherwise the epilogue and the LE stack walk
5507 // use a clobbered header (abend U4083). No signal may be delivered while
5508 // the stack pointer is changed and the header is copied, so this is done in
5509 // a stub routine (emitted once per module by the AsmPrinter), which gets
5510 // the new stack pointer in R1 and is called with BASR 3,3.
5511 if (Subtarget.isTargetXPLINK64()) {
5512 SDValue Glue;
5513 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: NewSP, Glue);
5514 Glue = Chain.getValue(R: 1);
5515 return DAG.getNode(Opcode: SystemZISD::XPLINK_STACKRESTORE, DL, VT: MVT::Other, N1: Chain,
5516 N2: Glue);
5517 }
5518
5519 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5520 SDValue Backchain;
5521 if (StoreBackchain) {
5522 SDValue OldSP = DAG.getCopyFromReg(
5523 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5524 Backchain =
5525 DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5526 PtrInfo: MachinePointerInfo());
5527 }
5528
5529 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5530
5531 if (StoreBackchain)
5532 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5533 PtrInfo: MachinePointerInfo());
5534
5535 return Chain;
5536}
5537
5538SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5539 SelectionDAG &DAG) const {
5540 bool IsData = Op.getConstantOperandVal(i: 4);
5541 if (!IsData)
5542 // Just preserve the chain.
5543 return Op.getOperand(i: 0);
5544
5545 SDLoc DL(Op);
5546 bool IsWrite = Op.getConstantOperandVal(i: 2);
5547 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5548 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5549 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5550 Op.getOperand(i: 1)};
5551 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5552 VTList: Node->getVTList(), Ops,
5553 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5554}
5555
5556SDValue
5557SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5558 SelectionDAG &DAG) const {
5559 unsigned Opcode, CCValid;
5560 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5561 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5562 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5563 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5564 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5565 return SDValue();
5566 }
5567
5568 return SDValue();
5569}
5570
5571SDValue
5572SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5573 SelectionDAG &DAG) const {
5574 unsigned Opcode, CCValid;
5575 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5576 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5577 if (Op->getNumValues() == 1)
5578 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5579 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5580 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5581 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5582 }
5583
5584 unsigned Id = Op.getConstantOperandVal(i: 0);
5585 switch (Id) {
5586 case Intrinsic::thread_pointer:
5587 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5588
5589 case Intrinsic::s390_vpdi:
5590 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5591 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5592
5593 case Intrinsic::s390_vperm:
5594 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5595 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5596
5597 case Intrinsic::s390_vuphb:
5598 case Intrinsic::s390_vuphh:
5599 case Intrinsic::s390_vuphf:
5600 case Intrinsic::s390_vuphg:
5601 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5602 Operand: Op.getOperand(i: 1));
5603
5604 case Intrinsic::s390_vuplhb:
5605 case Intrinsic::s390_vuplhh:
5606 case Intrinsic::s390_vuplhf:
5607 case Intrinsic::s390_vuplhg:
5608 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5609 Operand: Op.getOperand(i: 1));
5610
5611 case Intrinsic::s390_vuplb:
5612 case Intrinsic::s390_vuplhw:
5613 case Intrinsic::s390_vuplf:
5614 case Intrinsic::s390_vuplg:
5615 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5616 Operand: Op.getOperand(i: 1));
5617
5618 case Intrinsic::s390_vupllb:
5619 case Intrinsic::s390_vupllh:
5620 case Intrinsic::s390_vupllf:
5621 case Intrinsic::s390_vupllg:
5622 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5623 Operand: Op.getOperand(i: 1));
5624
5625 case Intrinsic::s390_vsumb:
5626 case Intrinsic::s390_vsumh:
5627 case Intrinsic::s390_vsumgh:
5628 case Intrinsic::s390_vsumgf:
5629 case Intrinsic::s390_vsumqf:
5630 case Intrinsic::s390_vsumqg:
5631 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5632 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5633
5634 case Intrinsic::s390_vaq:
5635 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5636 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5637 case Intrinsic::s390_vaccb:
5638 case Intrinsic::s390_vacch:
5639 case Intrinsic::s390_vaccf:
5640 case Intrinsic::s390_vaccg:
5641 case Intrinsic::s390_vaccq:
5642 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5643 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5644 case Intrinsic::s390_vacq:
5645 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5646 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5647 case Intrinsic::s390_vacccq:
5648 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5649 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5650
5651 case Intrinsic::s390_vsq:
5652 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5653 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5654 case Intrinsic::s390_vscbib:
5655 case Intrinsic::s390_vscbih:
5656 case Intrinsic::s390_vscbif:
5657 case Intrinsic::s390_vscbig:
5658 case Intrinsic::s390_vscbiq:
5659 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5660 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5661 case Intrinsic::s390_vsbiq:
5662 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5663 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5664 case Intrinsic::s390_vsbcbiq:
5665 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5666 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5667
5668 case Intrinsic::s390_vmhb:
5669 case Intrinsic::s390_vmhh:
5670 case Intrinsic::s390_vmhf:
5671 case Intrinsic::s390_vmhg:
5672 case Intrinsic::s390_vmhq:
5673 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5674 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5675 case Intrinsic::s390_vmlhb:
5676 case Intrinsic::s390_vmlhh:
5677 case Intrinsic::s390_vmlhf:
5678 case Intrinsic::s390_vmlhg:
5679 case Intrinsic::s390_vmlhq:
5680 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5681 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5682
5683 case Intrinsic::s390_vmahb:
5684 case Intrinsic::s390_vmahh:
5685 case Intrinsic::s390_vmahf:
5686 case Intrinsic::s390_vmahg:
5687 case Intrinsic::s390_vmahq:
5688 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5689 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5690 case Intrinsic::s390_vmalhb:
5691 case Intrinsic::s390_vmalhh:
5692 case Intrinsic::s390_vmalhf:
5693 case Intrinsic::s390_vmalhg:
5694 case Intrinsic::s390_vmalhq:
5695 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5696 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5697
5698 case Intrinsic::s390_vmeb:
5699 case Intrinsic::s390_vmeh:
5700 case Intrinsic::s390_vmef:
5701 case Intrinsic::s390_vmeg:
5702 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5703 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5704 case Intrinsic::s390_vmleb:
5705 case Intrinsic::s390_vmleh:
5706 case Intrinsic::s390_vmlef:
5707 case Intrinsic::s390_vmleg:
5708 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5709 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5710 case Intrinsic::s390_vmob:
5711 case Intrinsic::s390_vmoh:
5712 case Intrinsic::s390_vmof:
5713 case Intrinsic::s390_vmog:
5714 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5715 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5716 case Intrinsic::s390_vmlob:
5717 case Intrinsic::s390_vmloh:
5718 case Intrinsic::s390_vmlof:
5719 case Intrinsic::s390_vmlog:
5720 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5721 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5722
5723 case Intrinsic::s390_vmaeb:
5724 case Intrinsic::s390_vmaeh:
5725 case Intrinsic::s390_vmaef:
5726 case Intrinsic::s390_vmaeg:
5727 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5728 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5729 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5730 N2: Op.getOperand(i: 3));
5731 case Intrinsic::s390_vmaleb:
5732 case Intrinsic::s390_vmaleh:
5733 case Intrinsic::s390_vmalef:
5734 case Intrinsic::s390_vmaleg:
5735 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5736 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5737 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5738 N2: Op.getOperand(i: 3));
5739 case Intrinsic::s390_vmaob:
5740 case Intrinsic::s390_vmaoh:
5741 case Intrinsic::s390_vmaof:
5742 case Intrinsic::s390_vmaog:
5743 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5744 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5745 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5746 N2: Op.getOperand(i: 3));
5747 case Intrinsic::s390_vmalob:
5748 case Intrinsic::s390_vmaloh:
5749 case Intrinsic::s390_vmalof:
5750 case Intrinsic::s390_vmalog:
5751 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5752 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5753 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5754 N2: Op.getOperand(i: 3));
5755 }
5756
5757 return SDValue();
5758}
5759
5760namespace {
5761// Says that SystemZISD operation Opcode can be used to perform the equivalent
5762// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5763// Operand is the constant third operand, otherwise it is the number of
5764// bytes in each element of the result.
5765struct Permute {
5766 unsigned Opcode;
5767 unsigned Operand;
5768 unsigned char Bytes[SystemZ::VectorBytes];
5769};
5770}
5771
5772static const Permute PermuteForms[] = {
5773 // VMRHG
5774 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5775 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5776 // VMRHF
5777 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5778 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5779 // VMRHH
5780 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5781 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5782 // VMRHB
5783 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5784 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5785 // VMRLG
5786 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5787 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5788 // VMRLF
5789 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5790 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5791 // VMRLH
5792 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5793 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5794 // VMRLB
5795 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5796 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5797 // VPKG
5798 { .Opcode: SystemZISD::PACK, .Operand: 4,
5799 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5800 // VPKF
5801 { .Opcode: SystemZISD::PACK, .Operand: 2,
5802 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5803 // VPKH
5804 { .Opcode: SystemZISD::PACK, .Operand: 1,
5805 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5806 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5807 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5808 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5809 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5810 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5811 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5812};
5813
5814// Called after matching a vector shuffle against a particular pattern.
5815// Both the original shuffle and the pattern have two vector operands.
5816// OpNos[0] is the operand of the original shuffle that should be used for
5817// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5818// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5819// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5820// for operands 0 and 1 of the pattern.
5821static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5822 if (OpNos[0] < 0) {
5823 if (OpNos[1] < 0)
5824 return false;
5825 OpNo0 = OpNo1 = OpNos[1];
5826 } else if (OpNos[1] < 0) {
5827 OpNo0 = OpNo1 = OpNos[0];
5828 } else {
5829 OpNo0 = OpNos[0];
5830 OpNo1 = OpNos[1];
5831 }
5832 return true;
5833}
5834
5835// Bytes is a VPERM-like permute vector, except that -1 is used for
5836// undefined bytes. Return true if the VPERM can be implemented using P.
5837// When returning true set OpNo0 to the VPERM operand that should be
5838// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5839//
5840// For example, if swapping the VPERM operands allows P to match, OpNo0
5841// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5842// operand, but rewriting it to use two duplicated operands allows it to
5843// match P, then OpNo0 and OpNo1 will be the same.
5844static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5845 unsigned &OpNo0, unsigned &OpNo1) {
5846 int OpNos[] = { -1, -1 };
5847 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5848 int Elt = Bytes[I];
5849 if (Elt >= 0) {
5850 // Make sure that the two permute vectors use the same suboperand
5851 // byte number. Only the operand numbers (the high bits) are
5852 // allowed to differ.
5853 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5854 return false;
5855 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5856 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5857 // Make sure that the operand mappings are consistent with previous
5858 // elements.
5859 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5860 return false;
5861 OpNos[ModelOpNo] = RealOpNo;
5862 }
5863 }
5864 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5865}
5866
5867// As above, but search for a matching permute.
5868static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5869 unsigned &OpNo0, unsigned &OpNo1) {
5870 for (auto &P : PermuteForms)
5871 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5872 return &P;
5873 return nullptr;
5874}
5875
5876// Bytes is a VPERM-like permute vector, except that -1 is used for
5877// undefined bytes. This permute is an operand of an outer permute.
5878// See whether redistributing the -1 bytes gives a shuffle that can be
5879// implemented using P. If so, set Transform to a VPERM-like permute vector
5880// that, when applied to the result of P, gives the original permute in Bytes.
5881static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5882 const Permute &P,
5883 SmallVectorImpl<int> &Transform) {
5884 unsigned To = 0;
5885 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5886 int Elt = Bytes[From];
5887 if (Elt < 0)
5888 // Byte number From of the result is undefined.
5889 Transform[From] = -1;
5890 else {
5891 while (P.Bytes[To] != Elt) {
5892 To += 1;
5893 if (To == SystemZ::VectorBytes)
5894 return false;
5895 }
5896 Transform[From] = To;
5897 }
5898 }
5899 return true;
5900}
5901
5902// As above, but search for a matching permute.
5903static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5904 SmallVectorImpl<int> &Transform) {
5905 for (auto &P : PermuteForms)
5906 if (matchDoublePermute(Bytes, P, Transform))
5907 return &P;
5908 return nullptr;
5909}
5910
5911// Convert the mask of the given shuffle op into a byte-level mask,
5912// as if it had type vNi8.
5913static bool getVPermMask(SDValue ShuffleOp,
5914 SmallVectorImpl<int> &Bytes) {
5915 EVT VT = ShuffleOp.getValueType();
5916 unsigned NumElements = VT.getVectorNumElements();
5917 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5918
5919 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5920 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5921 for (unsigned I = 0; I < NumElements; ++I) {
5922 int Index = VSN->getMaskElt(Idx: I);
5923 if (Index >= 0)
5924 for (unsigned J = 0; J < BytesPerElement; ++J)
5925 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5926 }
5927 return true;
5928 }
5929 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5930 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5931 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5932 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5933 for (unsigned I = 0; I < NumElements; ++I)
5934 for (unsigned J = 0; J < BytesPerElement; ++J)
5935 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5936 return true;
5937 }
5938 return false;
5939}
5940
5941// Bytes is a VPERM-like permute vector, except that -1 is used for
5942// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5943// the result come from a contiguous sequence of bytes from one input.
5944// Set Base to the selector for the first byte if so.
5945static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5946 unsigned BytesPerElement, int &Base) {
5947 Base = -1;
5948 for (unsigned I = 0; I < BytesPerElement; ++I) {
5949 if (Bytes[Start + I] >= 0) {
5950 unsigned Elem = Bytes[Start + I];
5951 if (Base < 0) {
5952 Base = Elem - I;
5953 // Make sure the bytes would come from one input operand.
5954 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5955 return false;
5956 } else if (unsigned(Base) != Elem - I)
5957 return false;
5958 }
5959 }
5960 return true;
5961}
5962
5963// Bytes is a VPERM-like permute vector, except that -1 is used for
5964// undefined bytes. Return true if it can be performed using VSLDB.
5965// When returning true, set StartIndex to the shift amount and OpNo0
5966// and OpNo1 to the VPERM operands that should be used as the first
5967// and second shift operand respectively.
5968static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5969 unsigned &StartIndex, unsigned &OpNo0,
5970 unsigned &OpNo1) {
5971 int OpNos[] = { -1, -1 };
5972 int Shift = -1;
5973 for (unsigned I = 0; I < 16; ++I) {
5974 int Index = Bytes[I];
5975 if (Index >= 0) {
5976 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5977 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5978 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5979 if (Shift < 0)
5980 Shift = ExpectedShift;
5981 else if (Shift != ExpectedShift)
5982 return false;
5983 // Make sure that the operand mappings are consistent with previous
5984 // elements.
5985 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5986 return false;
5987 OpNos[ModelOpNo] = RealOpNo;
5988 }
5989 }
5990 StartIndex = Shift;
5991 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5992}
5993
5994// Create a node that performs P on operands Op0 and Op1, casting the
5995// operands to the appropriate type. The type of the result is determined by P.
5996static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5997 const Permute &P, SDValue Op0, SDValue Op1) {
5998 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5999 // elements of a PACK are twice as wide as the outputs.
6000 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
6001 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
6002 P.Operand);
6003 // Cast both operands to the appropriate type.
6004 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
6005 NumElements: SystemZ::VectorBytes / InBytes);
6006 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
6007 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
6008 SDValue Op;
6009 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
6010 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
6011 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
6012 } else if (P.Opcode == SystemZISD::PACK) {
6013 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
6014 NumElements: SystemZ::VectorBytes / P.Operand);
6015 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
6016 } else {
6017 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
6018 }
6019 return Op;
6020}
6021
6022static bool isZeroVector(SDValue N) {
6023 if (N->getOpcode() == ISD::BITCAST)
6024 N = N->getOperand(Num: 0);
6025 if (N->getOpcode() == ISD::SPLAT_VECTOR)
6026 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
6027 return Op->getZExtValue() == 0;
6028 return ISD::isBuildVectorAllZeros(V: N);
6029}
6030
6031// Return the index of the zero/undef vector, or UINT32_MAX if not found.
6032static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
6033 for (unsigned I = 0; I < Num ; I++)
6034 if (isZeroVector(N: Ops[I]))
6035 return I;
6036 return UINT32_MAX;
6037}
6038
6039// Bytes is a VPERM-like permute vector, except that -1 is used for
6040// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
6041// VSLDB or VPERM.
6042static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
6043 SDValue *Ops,
6044 const SmallVectorImpl<int> &Bytes) {
6045 for (unsigned I = 0; I < 2; ++I)
6046 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
6047
6048 // First see whether VSLDB can be used.
6049 unsigned StartIndex, OpNo0, OpNo1;
6050 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
6051 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
6052 N2: Ops[OpNo1],
6053 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
6054
6055 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
6056 // eliminate a zero vector by reusing any zero index in the permute vector.
6057 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
6058 if (ZeroVecIdx != UINT32_MAX) {
6059 bool MaskFirst = true;
6060 int ZeroIdx = -1;
6061 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
6062 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6063 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
6064 if (OpNo == ZeroVecIdx && I == 0) {
6065 // If the first byte is zero, use mask as first operand.
6066 ZeroIdx = 0;
6067 break;
6068 }
6069 if (OpNo != ZeroVecIdx && Byte == 0) {
6070 // If mask contains a zero, use it by placing that vector first.
6071 ZeroIdx = I + SystemZ::VectorBytes;
6072 MaskFirst = false;
6073 break;
6074 }
6075 }
6076 if (ZeroIdx != -1) {
6077 SDValue IndexNodes[SystemZ::VectorBytes];
6078 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
6079 if (Bytes[I] >= 0) {
6080 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6081 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
6082 if (OpNo == ZeroVecIdx)
6083 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
6084 else {
6085 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
6086 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
6087 }
6088 } else
6089 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
6090 }
6091 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
6092 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
6093 if (MaskFirst)
6094 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
6095 N3: Mask);
6096 else
6097 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
6098 N3: Mask);
6099 }
6100 }
6101
6102 SDValue IndexNodes[SystemZ::VectorBytes];
6103 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6104 if (Bytes[I] >= 0)
6105 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
6106 else
6107 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
6108 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
6109 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
6110 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
6111}
6112
6113namespace {
6114// Describes a general N-operand vector shuffle.
6115struct GeneralShuffle {
6116 GeneralShuffle(EVT vt)
6117 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
6118 void addUndef();
6119 bool add(SDValue, unsigned);
6120 SDValue getNode(SelectionDAG &, const SDLoc &);
6121 void tryPrepareForUnpack();
6122 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
6123 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
6124
6125 // The operands of the shuffle.
6126 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
6127
6128 // Index I is -1 if byte I of the result is undefined. Otherwise the
6129 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
6130 // Bytes[I] / SystemZ::VectorBytes.
6131 SmallVector<int, SystemZ::VectorBytes> Bytes;
6132
6133 // The type of the shuffle result.
6134 EVT VT;
6135
6136 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
6137 unsigned UnpackFromEltSize;
6138 // True if the final unpack uses the low half.
6139 bool UnpackLow;
6140};
6141} // namespace
6142
6143// Add an extra undefined element to the shuffle.
6144void GeneralShuffle::addUndef() {
6145 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6146 for (unsigned I = 0; I < BytesPerElement; ++I)
6147 Bytes.push_back(Elt: -1);
6148}
6149
6150// Add an extra element to the shuffle, taking it from element Elem of Op.
6151// A null Op indicates a vector input whose value will be calculated later;
6152// there is at most one such input per shuffle and it always has the same
6153// type as the result. Aborts and returns false if the source vector elements
6154// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
6155// LLVM they become implicitly extended, but this is rare and not optimized.
6156bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
6157 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6158
6159 // The source vector can have wider elements than the result,
6160 // either through an explicit TRUNCATE or because of type legalization.
6161 // We want the least significant part.
6162 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
6163 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
6164
6165 // Return false if the source elements are smaller than their destination
6166 // elements.
6167 if (FromBytesPerElement < BytesPerElement)
6168 return false;
6169
6170 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6171 (FromBytesPerElement - BytesPerElement));
6172
6173 // Look through things like shuffles and bitcasts.
6174 while (Op.getNode()) {
6175 if (Op.getOpcode() == ISD::BITCAST)
6176 Op = Op.getOperand(i: 0);
6177 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6178 // See whether the bytes we need come from a contiguous part of one
6179 // operand.
6180 SmallVector<int, SystemZ::VectorBytes> OpBytes;
6181 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
6182 break;
6183 int NewByte;
6184 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
6185 break;
6186 if (NewByte < 0) {
6187 addUndef();
6188 return true;
6189 }
6190 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
6191 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6192 } else if (Op.isUndef()) {
6193 addUndef();
6194 return true;
6195 } else
6196 break;
6197 }
6198
6199 // Make sure that the source of the extraction is in Ops.
6200 unsigned OpNo = 0;
6201 for (; OpNo < Ops.size(); ++OpNo)
6202 if (Ops[OpNo] == Op)
6203 break;
6204 if (OpNo == Ops.size())
6205 Ops.push_back(Elt: Op);
6206
6207 // Add the element to Bytes.
6208 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6209 for (unsigned I = 0; I < BytesPerElement; ++I)
6210 Bytes.push_back(Elt: Base + I);
6211
6212 return true;
6213}
6214
6215// Return SDNodes for the completed shuffle.
6216SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6217 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6218
6219 if (Ops.size() == 0)
6220 return DAG.getUNDEF(VT);
6221
6222 // Use a single unpack if possible as the last operation.
6223 tryPrepareForUnpack();
6224
6225 // Make sure that there are at least two shuffle operands.
6226 if (Ops.size() == 1)
6227 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
6228
6229 // Create a tree of shuffles, deferring root node until after the loop.
6230 // Try to redistribute the undefined elements of non-root nodes so that
6231 // the non-root shuffles match something like a pack or merge, then adjust
6232 // the parent node's permute vector to compensate for the new order.
6233 // Among other things, this copes with vectors like <2 x i16> that were
6234 // padded with undefined elements during type legalization.
6235 //
6236 // In the best case this redistribution will lead to the whole tree
6237 // using packs and merges. It should rarely be a loss in other cases.
6238 unsigned Stride = 1;
6239 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6240 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6241 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6242
6243 // Create a mask for just these two operands.
6244 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
6245 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6246 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6247 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6248 if (OpNo == I)
6249 NewBytes[J] = Byte;
6250 else if (OpNo == I + Stride)
6251 NewBytes[J] = SystemZ::VectorBytes + Byte;
6252 else
6253 NewBytes[J] = -1;
6254 }
6255 // See if it would be better to reorganize NewMask to avoid using VPERM.
6256 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6257 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6258 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6259 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6260 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6261 if (NewBytes[J] >= 0) {
6262 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6263 "Invalid double permute");
6264 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6265 } else
6266 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6267 }
6268 } else {
6269 // Just use NewBytes on the operands.
6270 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6271 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6272 if (NewBytes[J] >= 0)
6273 Bytes[J] = I * SystemZ::VectorBytes + J;
6274 }
6275 }
6276 }
6277
6278 // Now we just have 2 inputs. Put the second operand in Ops[1].
6279 if (Stride > 1) {
6280 Ops[1] = Ops[Stride];
6281 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6282 if (Bytes[I] >= int(SystemZ::VectorBytes))
6283 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6284 }
6285
6286 // Look for an instruction that can do the permute without resorting
6287 // to VPERM.
6288 unsigned OpNo0, OpNo1;
6289 SDValue Op;
6290 if (unpackWasPrepared() && Ops[1].isUndef())
6291 Op = Ops[0];
6292 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6293 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6294 else
6295 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6296
6297 Op = insertUnpackIfPrepared(DAG, DL, Op);
6298
6299 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6300}
6301
6302#ifndef NDEBUG
6303static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6304 dbgs() << Msg.c_str() << " { ";
6305 for (unsigned I = 0; I < Bytes.size(); I++)
6306 dbgs() << Bytes[I] << " ";
6307 dbgs() << "}\n";
6308}
6309#endif
6310
6311// If the Bytes vector matches an unpack operation, prepare to do the unpack
6312// after all else by removing the zero vector and the effect of the unpack on
6313// Bytes.
6314void GeneralShuffle::tryPrepareForUnpack() {
6315 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6316 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6317 return;
6318
6319 // Only do this if removing the zero vector reduces the depth, otherwise
6320 // the critical path will increase with the final unpack.
6321 if (Ops.size() > 2 &&
6322 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6323 return;
6324
6325 // Find an unpack that would allow removing the zero vector from Ops.
6326 UnpackFromEltSize = 1;
6327 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6328 bool MatchUnpack = true;
6329 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6330 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6331 unsigned ToEltSize = UnpackFromEltSize * 2;
6332 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6333 if (!IsZextByte)
6334 SrcBytes.push_back(Elt: Bytes[Elt]);
6335 if (Bytes[Elt] != -1) {
6336 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6337 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6338 MatchUnpack = false;
6339 break;
6340 }
6341 }
6342 }
6343 if (MatchUnpack) {
6344 if (Ops.size() == 2) {
6345 // Don't use unpack if a single source operand needs rearrangement.
6346 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6347 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6348 if (SrcBytes[i] == -1)
6349 continue;
6350 if (SrcBytes[i] % 16 != int(i))
6351 CanUseUnpackHigh = false;
6352 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6353 CanUseUnpackLow = false;
6354 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6355 UnpackFromEltSize = UINT_MAX;
6356 return;
6357 }
6358 }
6359 if (!CanUseUnpackHigh)
6360 UnpackLow = true;
6361 }
6362 break;
6363 }
6364 }
6365 if (UnpackFromEltSize > 4)
6366 return;
6367
6368 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6369 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6370 << ".\n";
6371 dumpBytes(Bytes, "Original Bytes vector:"););
6372
6373 // Apply the unpack in reverse to the Bytes array.
6374 unsigned B = 0;
6375 if (UnpackLow) {
6376 while (B < SystemZ::VectorBytes / 2)
6377 Bytes[B++] = -1;
6378 }
6379 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6380 Elt += UnpackFromEltSize;
6381 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6382 Bytes[B] = Bytes[Elt];
6383 }
6384 if (!UnpackLow) {
6385 while (B < SystemZ::VectorBytes)
6386 Bytes[B++] = -1;
6387 }
6388
6389 // Remove the zero vector from Ops
6390 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6391 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6392 if (Bytes[I] >= 0) {
6393 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6394 if (OpNo > ZeroVecOpNo)
6395 Bytes[I] -= SystemZ::VectorBytes;
6396 }
6397
6398 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6399 dbgs() << "\n";);
6400}
6401
6402SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6403 const SDLoc &DL,
6404 SDValue Op) {
6405 if (!unpackWasPrepared())
6406 return Op;
6407 unsigned InBits = UnpackFromEltSize * 8;
6408 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6409 NumElements: SystemZ::VectorBits / InBits);
6410 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6411 unsigned OutBits = InBits * 2;
6412 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6413 NumElements: SystemZ::VectorBits / OutBits);
6414 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6415 : SystemZISD::UNPACKL_HIGH,
6416 DL, VT: OutVT, Operand: PackedOp);
6417}
6418
6419// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6420static bool isScalarToVector(SDValue Op) {
6421 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6422 if (!Op.getOperand(i: I).isUndef())
6423 return false;
6424 return true;
6425}
6426
6427// Return a vector of type VT that contains Value in the first element.
6428// The other elements don't matter.
6429static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6430 SDValue Value) {
6431 // If we have a constant, replicate it to all elements and let the
6432 // BUILD_VECTOR lowering take care of it.
6433 if (Value.getOpcode() == ISD::Constant ||
6434 Value.getOpcode() == ISD::ConstantFP) {
6435 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6436 return DAG.getBuildVector(VT, DL, Ops);
6437 }
6438 if (Value.isUndef())
6439 return DAG.getUNDEF(VT);
6440 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6441}
6442
6443// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6444// element 1. Used for cases in which replication is cheap.
6445static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6446 SDValue Op0, SDValue Op1) {
6447 if (Op0.isUndef()) {
6448 if (Op1.isUndef())
6449 return DAG.getUNDEF(VT);
6450 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6451 }
6452 if (Op1.isUndef())
6453 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6454 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6455 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6456 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6457}
6458
6459// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6460// vector for them.
6461static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6462 SDValue Op1) {
6463 if (Op0.isUndef() && Op1.isUndef())
6464 return DAG.getUNDEF(VT: MVT::v2i64);
6465 // If one of the two inputs is undefined then replicate the other one,
6466 // in order to avoid using another register unnecessarily.
6467 if (Op0.isUndef())
6468 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6469 else if (Op1.isUndef())
6470 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6471 else {
6472 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6473 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6474 }
6475 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6476}
6477
6478// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6479// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6480// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6481// would benefit from this representation and return it if so.
6482static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6483 BuildVectorSDNode *BVN) {
6484 EVT VT = BVN->getValueType(ResNo: 0);
6485 unsigned NumElements = VT.getVectorNumElements();
6486
6487 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6488 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6489 // need a BUILD_VECTOR, add an additional placeholder operand for that
6490 // BUILD_VECTOR and store its operands in ResidueOps.
6491 GeneralShuffle GS(VT);
6492 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6493 bool FoundOne = false;
6494 for (unsigned I = 0; I < NumElements; ++I) {
6495 SDValue Op = BVN->getOperand(Num: I);
6496 if (Op.getOpcode() == ISD::TRUNCATE)
6497 Op = Op.getOperand(i: 0);
6498 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6499 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6500 unsigned Elem = Op.getConstantOperandVal(i: 1);
6501 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6502 return SDValue();
6503 FoundOne = true;
6504 } else if (Op.isUndef()) {
6505 GS.addUndef();
6506 } else {
6507 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6508 return SDValue();
6509 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6510 }
6511 }
6512
6513 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6514 if (!FoundOne)
6515 return SDValue();
6516
6517 // Create the BUILD_VECTOR for the remaining elements, if any.
6518 if (!ResidueOps.empty()) {
6519 while (ResidueOps.size() < NumElements)
6520 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6521 for (auto &Op : GS.Ops) {
6522 if (!Op.getNode()) {
6523 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6524 break;
6525 }
6526 }
6527 }
6528 return GS.getNode(DAG, DL: SDLoc(BVN));
6529}
6530
6531bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6532 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6533 return true;
6534 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6535 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6536 return true;
6537 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6538 return true;
6539 return false;
6540}
6541
6542static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL,
6543 unsigned MergedBits, EVT VT, SDValue Op0,
6544 SDValue Op1) {
6545 MVT IntVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MergedBits),
6546 NumElements: SystemZ::VectorBits / MergedBits);
6547 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6548 "Handling full vectors only.");
6549 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0);
6550 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op1);
6551 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT: IntVecVT, N1: Op0, N2: Op1);
6552 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6553}
6554
6555static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL,
6556 EVT VT, SmallVectorImpl<SDValue> &Elems,
6557 unsigned Pos) {
6558 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 0], Op1: Elems[Pos + 1]);
6559 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 2], Op1: Elems[Pos + 3]);
6560 // Avoid unnecessary undefs by reusing the other operand.
6561 if (Op01.isUndef()) {
6562 if (Op23.isUndef())
6563 return Op01;
6564 Op01 = Op23;
6565 } else if (Op23.isUndef())
6566 Op23 = Op01;
6567 // Merging identical replications is a no-op.
6568 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6569 return Op01;
6570 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6571 return mergeHighParts(DAG, DL, MergedBits, VT, Op0: Op01, Op1: Op23);
6572}
6573
6574// Combine GPR scalar values Elems into a vector of type VT.
6575SDValue
6576SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6577 SmallVectorImpl<SDValue> &Elems) const {
6578 // See whether there is a single replicated value.
6579 SDValue Single;
6580 unsigned int NumElements = Elems.size();
6581 unsigned int Count = 0;
6582 for (auto Elem : Elems) {
6583 if (!Elem.isUndef()) {
6584 if (!Single.getNode())
6585 Single = Elem;
6586 else if (Elem != Single) {
6587 Single = SDValue();
6588 break;
6589 }
6590 Count += 1;
6591 }
6592 }
6593 // There are three cases here:
6594 //
6595 // - if the only defined element is a loaded one, the best sequence
6596 // is a replicating load.
6597 //
6598 // - otherwise, if the only defined element is an i64 value, we will
6599 // end up with the same VLVGP sequence regardless of whether we short-cut
6600 // for replication or fall through to the later code.
6601 //
6602 // - otherwise, if the only defined element is an i32 or smaller value,
6603 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6604 // This is only a win if the single defined element is used more than once.
6605 // In other cases we're better off using a single VLVGx.
6606 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6607 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6608
6609 // If all elements are loads, use VLREP/VLEs (below).
6610 bool AllLoads = true;
6611 for (auto Elem : Elems)
6612 if (!isVectorElementLoad(Op: Elem)) {
6613 AllLoads = false;
6614 break;
6615 }
6616
6617 // The best way of building a v2i64 from two i64s is to use VLVGP.
6618 if (VT == MVT::v2i64 && !AllLoads)
6619 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6620
6621 // Use a 64-bit merge high to combine two doubles.
6622 if (VT == MVT::v2f64 && !AllLoads)
6623 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6624
6625 // Build v4f32 values directly from the FPRs:
6626 //
6627 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6628 // V V VMRHF
6629 // <ABxx> <CDxx>
6630 // V VMRHG
6631 // <ABCD>
6632 if (VT == MVT::v4f32 && !AllLoads)
6633 return buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6634
6635 // Same for v8f16.
6636 if (VT == MVT::v8f16 && !AllLoads) {
6637 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6638 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 4);
6639 // Avoid unnecessary undefs by reusing the other operand.
6640 if (Op0123.isUndef())
6641 Op0123 = Op4567;
6642 else if (Op4567.isUndef())
6643 Op4567 = Op0123;
6644 // Merging identical replications is a no-op.
6645 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6646 return Op0123;
6647 return mergeHighParts(DAG, DL, MergedBits: 64, VT, Op0: Op0123, Op1: Op4567);
6648 }
6649
6650 // Collect the constant terms.
6651 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6652 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6653
6654 unsigned NumConstants = 0;
6655 for (unsigned I = 0; I < NumElements; ++I) {
6656 SDValue Elem = Elems[I];
6657 if (Elem.getOpcode() == ISD::Constant ||
6658 Elem.getOpcode() == ISD::ConstantFP) {
6659 NumConstants += 1;
6660 Constants[I] = Elem;
6661 Done[I] = true;
6662 }
6663 }
6664 // If there was at least one constant, fill in the other elements of
6665 // Constants with undefs to get a full vector constant and use that
6666 // as the starting point.
6667 SDValue Result;
6668 SDValue ReplicatedVal;
6669 if (NumConstants > 0) {
6670 for (unsigned I = 0; I < NumElements; ++I)
6671 if (!Constants[I].getNode())
6672 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6673 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6674 } else {
6675 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6676 // avoid a false dependency on any previous contents of the vector
6677 // register.
6678
6679 // Use a VLREP if at least one element is a load. Make sure to replicate
6680 // the load with the most elements having its value.
6681 std::map<const SDNode*, unsigned> UseCounts;
6682 SDNode *LoadMaxUses = nullptr;
6683 for (unsigned I = 0; I < NumElements; ++I)
6684 if (isVectorElementLoad(Op: Elems[I])) {
6685 SDNode *Ld = Elems[I].getNode();
6686 unsigned Count = ++UseCounts[Ld];
6687 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6688 LoadMaxUses = Ld;
6689 }
6690 if (LoadMaxUses != nullptr) {
6691 ReplicatedVal = SDValue(LoadMaxUses, 0);
6692 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6693 } else {
6694 // Try to use VLVGP.
6695 unsigned I1 = NumElements / 2 - 1;
6696 unsigned I2 = NumElements - 1;
6697 bool Def1 = !Elems[I1].isUndef();
6698 bool Def2 = !Elems[I2].isUndef();
6699 if (Def1 || Def2) {
6700 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6701 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6702 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6703 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6704 Done[I1] = true;
6705 Done[I2] = true;
6706 } else
6707 Result = DAG.getUNDEF(VT);
6708 }
6709 }
6710
6711 // Use VLVGx to insert the other elements.
6712 for (unsigned I = 0; I < NumElements; ++I)
6713 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6714 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6715 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6716 return Result;
6717}
6718
6719SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6720 SelectionDAG &DAG) const {
6721 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6722 SDLoc DL(Op);
6723 EVT VT = Op.getValueType();
6724
6725 if (BVN->isConstant()) {
6726 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6727 return Op;
6728
6729 // Fall back to loading it from memory.
6730 return SDValue();
6731 }
6732
6733 // See if we should use shuffles to construct the vector from other vectors.
6734 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6735 return Res;
6736
6737 // Detect SCALAR_TO_VECTOR conversions.
6738 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6739 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6740
6741 // Otherwise use buildVector to build the vector up from GPRs.
6742 unsigned NumElements = Op.getNumOperands();
6743 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6744 for (unsigned I = 0; I < NumElements; ++I)
6745 Ops[I] = Op.getOperand(i: I);
6746 return buildVector(DAG, DL, VT, Elems&: Ops);
6747}
6748
6749SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6750 SelectionDAG &DAG) const {
6751 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6752 SDLoc DL(Op);
6753 EVT VT = Op.getValueType();
6754 unsigned NumElements = VT.getVectorNumElements();
6755
6756 if (VSN->isSplat()) {
6757 SDValue Op0 = Op.getOperand(i: 0);
6758 unsigned Index = VSN->getSplatIndex();
6759 assert(Index < VT.getVectorNumElements() &&
6760 "Splat index should be defined and in first operand");
6761 // See whether the value we're splatting is directly available as a scalar.
6762 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6763 Op0.getOpcode() == ISD::BUILD_VECTOR)
6764 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6765 // Otherwise keep it as a vector-to-vector operation.
6766 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6767 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6768 }
6769
6770 GeneralShuffle GS(VT);
6771 for (unsigned I = 0; I < NumElements; ++I) {
6772 int Elt = VSN->getMaskElt(Idx: I);
6773 if (Elt < 0)
6774 GS.addUndef();
6775 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6776 Elem: unsigned(Elt) % NumElements))
6777 return SDValue();
6778 }
6779 return GS.getNode(DAG, DL: SDLoc(VSN));
6780}
6781
6782SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6783 SelectionDAG &DAG) const {
6784 SDLoc DL(Op);
6785 // Just insert the scalar into element 0 of an undefined vector.
6786 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6787 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6788 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6789}
6790
6791// Shift the lower 2 bytes of Op to the left in order to insert into the
6792// upper 2 bytes of the FP register.
6793static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6794 assert(Op.getSimpleValueType() == MVT::i64 &&
6795 "Expexted to convert i64 to f16.");
6796 SDLoc DL(Op);
6797 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6798 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6799 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6800 SDValue F16Val =
6801 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6802 return F16Val;
6803}
6804
6805// Extract Op into GPR and shift the 2 f16 bytes to the right.
6806static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6807 assert(Op.getSimpleValueType() == MVT::f16 &&
6808 "Expected to convert f16 to i64.");
6809 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6810 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6811 Operand: SDValue(U32, 0), Subreg: Op);
6812 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6813 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6814 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6815 return Shft;
6816}
6817
6818static bool isInBoundsConstantVectorIndex(EVT VecVT, SDValue Idx) {
6819 auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
6820 return CIdx && CIdx->getZExtValue() < VecVT.getVectorNumElements();
6821}
6822
6823SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6824 SelectionDAG &DAG) const {
6825 // Handle insertions of floating-point values.
6826 SDLoc DL(Op);
6827 SDValue Op0 = Op.getOperand(i: 0);
6828 SDValue Op1 = Op.getOperand(i: 1);
6829 SDValue Op2 = Op.getOperand(i: 2);
6830 EVT VT = Op.getValueType();
6831
6832 // A constant index selects a GR128 subregister.
6833 if (!Subtarget.hasVector())
6834 return isInBoundsConstantVectorIndex(VecVT: VT, Idx: Op2) ? Op : SDValue();
6835
6836 // Insertions into constant indices of a v2f64 can be done using VPDI.
6837 // However, if the inserted value is a bitcast or a constant then it's
6838 // better to use GPRs, as below.
6839 if (VT == MVT::v2f64 && Op1.getOpcode() != ISD::BITCAST &&
6840 Op1.getOpcode() != ISD::ConstantFP &&
6841 isInBoundsConstantVectorIndex(VecVT: VT, Idx: Op2))
6842 return Op;
6843
6844 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6845 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6846 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6847 SDValue IntOp1 =
6848 VT == MVT::v8f16
6849 ? DAG.getZExtOrTrunc(Op: convertFromF16(Op: Op1, DL, DAG), DL, VT: MVT::i32)
6850 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1);
6851 SDValue Res =
6852 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6853 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: IntOp1, N3: Op2);
6854 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6855}
6856
6857SDValue
6858SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6859 SelectionDAG &DAG) const {
6860 // Handle extractions of floating-point values.
6861 SDLoc DL(Op);
6862 SDValue Op0 = Op.getOperand(i: 0);
6863 SDValue Op1 = Op.getOperand(i: 1);
6864 EVT VT = Op.getValueType();
6865 EVT VecVT = Op0.getValueType();
6866
6867 // Extractions of constant indices can be done directly.
6868 if (isInBoundsConstantVectorIndex(VecVT, Idx: Op1))
6869 return Op;
6870
6871 if (!Subtarget.hasVector())
6872 return SDValue();
6873
6874 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6875 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6876 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6877 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6878 SDValue Extr = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ExtrVT,
6879 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6880 if (VT == MVT::f16)
6881 return convertToF16(Op: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Extr), DAG);
6882 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Extr);
6883}
6884
6885SDValue SystemZTargetLowering::
6886lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6887 SDValue PackedOp = Op.getOperand(i: 0);
6888 EVT OutVT = Op.getValueType();
6889 EVT InVT = PackedOp.getValueType();
6890 unsigned ToBits = OutVT.getScalarSizeInBits();
6891 unsigned FromBits = InVT.getScalarSizeInBits();
6892 unsigned StartOffset = 0;
6893
6894 // If the input is a VECTOR_SHUFFLE, there are a number of important
6895 // cases where we can directly implement the sign-extension of the
6896 // original input lanes of the shuffle.
6897 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6898 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6899 ArrayRef<int> ShuffleMask = SVN->getMask();
6900 int OutNumElts = OutVT.getVectorNumElements();
6901
6902 // Recognize the special case where the sign-extension can be done
6903 // by the VSEG instruction. Handled via the default expander.
6904 if (ToBits == 64 && OutNumElts == 2) {
6905 int NumElem = ToBits / FromBits;
6906 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6907 return SDValue();
6908 }
6909
6910 // Recognize the special case where we can fold the shuffle by
6911 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6912 int StartOffsetCandidate = -1;
6913 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6914 if (ShuffleMask[Elt] == -1)
6915 continue;
6916 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6917 if (StartOffsetCandidate == -1)
6918 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6919 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6920 continue;
6921 }
6922 StartOffsetCandidate = -1;
6923 break;
6924 }
6925 if (StartOffsetCandidate != -1) {
6926 // The mask indexes the concatenation of both shuffle operands: an offset
6927 // past the first operand's lanes selects lanes of the second.
6928 unsigned InNumElts = InVT.getVectorNumElements();
6929 unsigned Offset = StartOffsetCandidate;
6930 unsigned OpNo = Offset >= InNumElts ? 1 : 0;
6931 StartOffset = Offset - OpNo * InNumElts;
6932 PackedOp = PackedOp.getOperand(i: OpNo);
6933 }
6934 }
6935
6936 do {
6937 FromBits *= 2;
6938 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6939 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6940 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6941 if (StartOffset >= OutNumElts) {
6942 Opcode = SystemZISD::UNPACK_LOW;
6943 StartOffset -= OutNumElts;
6944 }
6945 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6946 } while (FromBits != ToBits);
6947 return PackedOp;
6948}
6949
6950// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6951SDValue SystemZTargetLowering::
6952lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6953 SDValue PackedOp = Op.getOperand(i: 0);
6954 SDLoc DL(Op);
6955 EVT OutVT = Op.getValueType();
6956 EVT InVT = PackedOp.getValueType();
6957 unsigned InNumElts = InVT.getVectorNumElements();
6958 unsigned OutNumElts = OutVT.getVectorNumElements();
6959 unsigned NumInPerOut = InNumElts / OutNumElts;
6960
6961 SDValue ZeroVec =
6962 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6963
6964 SmallVector<int, 16> Mask(InNumElts);
6965 unsigned ZeroVecElt = InNumElts;
6966 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6967 unsigned MaskElt = PackedElt * NumInPerOut;
6968 unsigned End = MaskElt + NumInPerOut - 1;
6969 for (; MaskElt < End; MaskElt++)
6970 Mask[MaskElt] = ZeroVecElt++;
6971 Mask[MaskElt] = PackedElt;
6972 }
6973 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6974 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6975}
6976
6977SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6978 unsigned ByScalar) const {
6979 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6980 SDValue Op0 = Op.getOperand(i: 0);
6981 SDValue Op1 = Op.getOperand(i: 1);
6982 SDLoc DL(Op);
6983 EVT VT = Op.getValueType();
6984 unsigned ElemBitSize = VT.getScalarSizeInBits();
6985
6986 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6987 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6988 APInt SplatBits, SplatUndef;
6989 unsigned SplatBitSize;
6990 bool HasAnyUndefs;
6991 // Check for constant splats. Use ElemBitSize as the minimum element
6992 // width and reject splats that need wider elements.
6993 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6994 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6995 SplatBitSize == ElemBitSize) {
6996 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6997 DL, VT: MVT::i32);
6998 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6999 }
7000 // Check for variable splats.
7001 BitVector UndefElements;
7002 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
7003 if (Splat) {
7004 // Since i32 is the smallest legal type, we either need a no-op
7005 // or a truncation.
7006 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
7007 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
7008 }
7009 }
7010
7011 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
7012 // and the shift amount is directly available in a GPR.
7013 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
7014 if (VSN->isSplat()) {
7015 SDValue VSNOp0 = VSN->getOperand(Num: 0);
7016 unsigned Index = VSN->getSplatIndex();
7017 assert(Index < VT.getVectorNumElements() &&
7018 "Splat index should be defined and in first operand");
7019 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
7020 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
7021 // Since i32 is the smallest legal type, we either need a no-op
7022 // or a truncation.
7023 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
7024 Operand: VSNOp0.getOperand(i: Index));
7025 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
7026 }
7027 }
7028 }
7029
7030 // Otherwise just treat the current form as legal.
7031 return Op;
7032}
7033
7034SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
7035 SDLoc DL(Op);
7036
7037 // i128 FSHL with a constant amount that is a multiple of 8 can be
7038 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
7039 // facility, FSHL with a constant amount less than 8 can be implemented
7040 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
7041 // combination of the two.
7042 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
7043 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7044 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7045 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
7046 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
7047 if (ShiftAmt > 120) {
7048 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
7049 // SHR_DOUBLE_BIT emits fewer instructions.
7050 SDValue Val =
7051 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
7052 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
7053 return DAG.getBitcast(VT: MVT::i128, V: Val);
7054 }
7055 SmallVector<int, 16> Mask(16);
7056 for (unsigned Elt = 0; Elt < 16; Elt++)
7057 Mask[Elt] = (ShiftAmt >> 3) + Elt;
7058 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
7059 if ((ShiftAmt & 7) == 0)
7060 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
7061 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
7062 SDValue Val =
7063 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
7064 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
7065 return DAG.getBitcast(VT: MVT::i128, V: Val);
7066 }
7067 }
7068
7069 return SDValue();
7070}
7071
7072SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
7073 SDLoc DL(Op);
7074
7075 // i128 FSHR with a constant amount that is a multiple of 8 can be
7076 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
7077 // facility, FSHR with a constant amount less than 8 can be implemented
7078 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
7079 // combination of the two.
7080 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
7081 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
7082 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
7083 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
7084 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
7085 if (ShiftAmt > 120) {
7086 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
7087 // SHL_DOUBLE_BIT emits fewer instructions.
7088 SDValue Val =
7089 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
7090 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
7091 return DAG.getBitcast(VT: MVT::i128, V: Val);
7092 }
7093 SmallVector<int, 16> Mask(16);
7094 for (unsigned Elt = 0; Elt < 16; Elt++)
7095 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
7096 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
7097 if ((ShiftAmt & 7) == 0)
7098 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
7099 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
7100 SDValue Val =
7101 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
7102 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
7103 return DAG.getBitcast(VT: MVT::i128, V: Val);
7104 }
7105 }
7106
7107 return SDValue();
7108}
7109
7110static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
7111 SDLoc DL(Op);
7112 SDValue Src = Op.getOperand(i: 0);
7113 MVT DstVT = Op.getSimpleValueType();
7114
7115 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
7116 unsigned SrcAS = N->getSrcAddressSpace();
7117
7118 assert(SrcAS != N->getDestAddressSpace() &&
7119 "addrspacecast must be between different address spaces");
7120
7121 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
7122 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
7123 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
7124 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
7125 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7126 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7127 } else if (DstVT == MVT::i32) {
7128 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
7129 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
7130 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7131 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7132 } else {
7133 report_fatal_error(reason: "Bad address space in addrspacecast");
7134 }
7135 return Op;
7136}
7137
7138SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
7139 SelectionDAG &DAG) const {
7140 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
7141 if (In.getSimpleValueType() != MVT::f16)
7142 return Op; // Legal
7143 return SDValue(); // Let legalizer emit the libcall.
7144}
7145
7146SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
7147 MVT VT, SDValue Arg, SDLoc DL,
7148 SDValue Chain, bool IsStrict) const {
7149 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
7150 MakeLibCallOptions CallOptions;
7151 SDValue Result;
7152 std::tie(args&: Result, args&: Chain) =
7153 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
7154 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
7155}
7156
7157SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
7158 SelectionDAG &DAG) const {
7159 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
7160 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
7161 bool IsStrict = Op->isStrictFPOpcode();
7162 SDLoc DL(Op);
7163 MVT VT = Op.getSimpleValueType();
7164 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7165 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7166 EVT InVT = InOp.getValueType();
7167
7168 // FP to unsigned is not directly supported on z10. Promoting an i32
7169 // result to (signed) i64 doesn't generate an inexact condition (fp
7170 // exception) for values that are outside the i32 range but in the i64
7171 // range, so use the default expansion.
7172 if (!Subtarget.hasFPExtension() && !IsSigned)
7173 // Expand i32/i64. F16 values will be recognized to fit and extended.
7174 return SDValue();
7175
7176 // Conversion from f16 is done via f32.
7177 if (InOp.getSimpleValueType() == MVT::f16) {
7178 SmallVector<SDValue, 2> Results;
7179 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7180 return DAG.getMergeValues(Ops: Results, dl: DL);
7181 }
7182
7183 if (VT == MVT::i128) {
7184 RTLIB::Libcall LC =
7185 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
7186 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7187 }
7188
7189 return Op; // Legal
7190}
7191
7192SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7193 SelectionDAG &DAG) const {
7194 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7195 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7196 bool IsStrict = Op->isStrictFPOpcode();
7197 SDLoc DL(Op);
7198 MVT VT = Op.getSimpleValueType();
7199 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7200 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7201 EVT InVT = InOp.getValueType();
7202
7203 // Conversion to f16 is done via f32.
7204 if (VT == MVT::f16) {
7205 SmallVector<SDValue, 2> Results;
7206 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7207 return DAG.getMergeValues(Ops: Results, dl: DL);
7208 }
7209
7210 // Unsigned to fp is not directly supported on z10.
7211 if (!Subtarget.hasFPExtension() && !IsSigned)
7212 return SDValue(); // Expand i64.
7213
7214 if (InVT == MVT::i128) {
7215 RTLIB::Libcall LC =
7216 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
7217 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7218 }
7219
7220 return Op; // Legal
7221}
7222
7223// Lower an f16 LOAD in case of no vector support.
7224SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7225 SelectionDAG &DAG) const {
7226 EVT RegVT = Op.getValueType();
7227 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7228 (void)RegVT;
7229
7230 // Load as integer.
7231 SDLoc DL(Op);
7232 SDValue NewLd;
7233 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
7234 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7235 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
7236 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
7237 MMO: AtomicLd->getMemOperand());
7238 } else {
7239 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
7240 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7241 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
7242 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
7243 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
7244 }
7245 SDValue F16Val = convertToF16(Op: NewLd, DAG);
7246 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
7247}
7248
7249// Lower an f16 STORE in case of no vector support.
7250SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7251 SelectionDAG &DAG) const {
7252 SDLoc DL(Op);
7253 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
7254
7255 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
7256 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
7257 Ptr: Shft, Val: AtomicSt->getBasePtr(),
7258 MMO: AtomicSt->getMemOperand());
7259
7260 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
7261 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
7262 MMO: St->getMemOperand());
7263}
7264
7265SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7266 SelectionDAG &DAG) const {
7267 SDLoc DL(Op);
7268 MVT ResultVT = Op.getSimpleValueType();
7269 SDValue Arg = Op.getOperand(i: 0);
7270 unsigned Check = Op.getConstantOperandVal(i: 1);
7271
7272 unsigned TDCMask = 0;
7273 if (Check & fcSNan)
7274 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
7275 if (Check & fcQNan)
7276 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
7277 if (Check & fcPosInf)
7278 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
7279 if (Check & fcNegInf)
7280 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
7281 if (Check & fcPosNormal)
7282 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
7283 if (Check & fcNegNormal)
7284 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
7285 if (Check & fcPosSubnormal)
7286 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
7287 if (Check & fcNegSubnormal)
7288 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
7289 if (Check & fcPosZero)
7290 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7291 if (Check & fcNegZero)
7292 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7293 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
7294
7295 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
7296 return getCCResult(DAG, CCReg: Intr);
7297}
7298
7299SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7300 SelectionDAG &DAG) const {
7301 SDLoc DL(Op);
7302 SDValue Chain = Op.getOperand(i: 0);
7303
7304 // STCKF only supports a memory operand, so we have to use a temporary.
7305 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
7306 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7307 MachinePointerInfo MPI =
7308 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7309
7310 // Use STCFK to store the TOD clock into the temporary.
7311 SDValue StoreOps[] = {Chain, StackPtr};
7312 Chain = DAG.getMemIntrinsicNode(
7313 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7314 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7315
7316 // And read it back from there.
7317 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7318}
7319
7320SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7321 SelectionDAG &DAG) const {
7322 switch (Op.getOpcode()) {
7323 case ISD::FRAMEADDR:
7324 return lowerFRAMEADDR(Op, DAG);
7325 case ISD::RETURNADDR:
7326 return lowerRETURNADDR(Op, DAG);
7327 case ISD::BR_CC:
7328 return lowerBR_CC(Op, DAG);
7329 case ISD::SELECT_CC:
7330 return lowerSELECT_CC(Op, DAG);
7331 case ISD::SETCC:
7332 return lowerSETCC(Op, DAG);
7333 case ISD::STRICT_FSETCC:
7334 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7335 case ISD::STRICT_FSETCCS:
7336 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7337 case ISD::GlobalAddress:
7338 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7339 case ISD::GlobalTLSAddress:
7340 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7341 case ISD::BlockAddress:
7342 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7343 case ISD::JumpTable:
7344 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7345 case ISD::ConstantPool:
7346 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7347 case ISD::BITCAST:
7348 return lowerBITCAST(Op, DAG);
7349 case ISD::VASTART:
7350 return lowerVASTART(Op, DAG);
7351 case ISD::VACOPY:
7352 return lowerVACOPY(Op, DAG);
7353 case ISD::DYNAMIC_STACKALLOC:
7354 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7355 case ISD::GET_DYNAMIC_AREA_OFFSET:
7356 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7357 case ISD::MULHS:
7358 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7359 case ISD::MULHU:
7360 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7361 case ISD::SMUL_LOHI:
7362 return lowerSMUL_LOHI(Op, DAG);
7363 case ISD::UMUL_LOHI:
7364 return lowerUMUL_LOHI(Op, DAG);
7365 case ISD::SDIVREM:
7366 return lowerSDIVREM(Op, DAG);
7367 case ISD::UDIVREM:
7368 return lowerUDIVREM(Op, DAG);
7369 case ISD::SADDO:
7370 case ISD::SSUBO:
7371 case ISD::UADDO:
7372 case ISD::USUBO:
7373 return lowerXALUO(Op, DAG);
7374 case ISD::UADDO_CARRY:
7375 case ISD::USUBO_CARRY:
7376 return lowerUADDSUBO_CARRY(Op, DAG);
7377 case ISD::OR:
7378 return lowerOR(Op, DAG);
7379 case ISD::CTPOP:
7380 return lowerCTPOP(Op, DAG);
7381 case ISD::VECREDUCE_ADD:
7382 return lowerVECREDUCE_ADD(Op, DAG);
7383 case ISD::ATOMIC_FENCE:
7384 return lowerATOMIC_FENCE(Op, DAG);
7385 case ISD::ATOMIC_STORE:
7386 return lowerATOMIC_STORE(Op, DAG);
7387 case ISD::ATOMIC_LOAD:
7388 return lowerATOMIC_LOAD(Op, DAG);
7389 case ISD::ATOMIC_LOAD_SUB:
7390 return lowerATOMIC_LOAD_SUB(Op, DAG);
7391 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7392 return lowerATOMIC_CMP_SWAP(Op, DAG);
7393 case ISD::STACKSAVE:
7394 return lowerSTACKSAVE(Op, DAG);
7395 case ISD::STACKRESTORE:
7396 return lowerSTACKRESTORE(Op, DAG);
7397 case ISD::PREFETCH:
7398 return lowerPREFETCH(Op, DAG);
7399 case ISD::INTRINSIC_W_CHAIN:
7400 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7401 case ISD::INTRINSIC_WO_CHAIN:
7402 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7403 case ISD::BUILD_VECTOR:
7404 return lowerBUILD_VECTOR(Op, DAG);
7405 case ISD::VECTOR_SHUFFLE:
7406 return lowerVECTOR_SHUFFLE(Op, DAG);
7407 case ISD::SCALAR_TO_VECTOR:
7408 return lowerSCALAR_TO_VECTOR(Op, DAG);
7409 case ISD::INSERT_VECTOR_ELT:
7410 return lowerINSERT_VECTOR_ELT(Op, DAG);
7411 case ISD::EXTRACT_VECTOR_ELT:
7412 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7413 case ISD::SIGN_EXTEND_VECTOR_INREG:
7414 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7415 case ISD::ZERO_EXTEND_VECTOR_INREG:
7416 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7417 case ISD::SHL:
7418 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7419 case ISD::SRL:
7420 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7421 case ISD::SRA:
7422 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7423 case ISD::ADDRSPACECAST:
7424 return lowerAddrSpaceCast(Op, DAG);
7425 case ISD::ROTL:
7426 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7427 case ISD::FSHL:
7428 return lowerFSHL(Op, DAG);
7429 case ISD::FSHR:
7430 return lowerFSHR(Op, DAG);
7431 case ISD::FP_EXTEND:
7432 case ISD::STRICT_FP_EXTEND:
7433 return lowerFP_EXTEND(Op, DAG);
7434 case ISD::FP_TO_UINT:
7435 case ISD::FP_TO_SINT:
7436 case ISD::STRICT_FP_TO_UINT:
7437 case ISD::STRICT_FP_TO_SINT:
7438 return lower_FP_TO_INT(Op, DAG);
7439 case ISD::UINT_TO_FP:
7440 case ISD::SINT_TO_FP:
7441 case ISD::STRICT_UINT_TO_FP:
7442 case ISD::STRICT_SINT_TO_FP:
7443 return lower_INT_TO_FP(Op, DAG);
7444 case ISD::LOAD:
7445 return lowerLoadF16(Op, DAG);
7446 case ISD::STORE:
7447 return lowerStoreF16(Op, DAG);
7448 case ISD::IS_FPCLASS:
7449 return lowerIS_FPCLASS(Op, DAG);
7450 case ISD::GET_ROUNDING:
7451 return lowerGET_ROUNDING(Op, DAG);
7452 case ISD::READCYCLECOUNTER:
7453 return lowerREADCYCLECOUNTER(Op, DAG);
7454 case ISD::EH_SJLJ_SETJMP:
7455 case ISD::EH_SJLJ_LONGJMP:
7456 // These operations are legal on our platform, but we cannot actually
7457 // set the operation action to Legal as common code would treat this
7458 // as equivalent to Expand. Instead, we keep the operation action to
7459 // Custom and just leave them unchanged here.
7460 return Op;
7461
7462 default:
7463 llvm_unreachable("Unexpected node to lower");
7464 }
7465}
7466
7467static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7468 const SDLoc &SL) {
7469 // If i128 is legal, just use a normal bitcast.
7470 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7471 return DAG.getBitcast(VT: MVT::f128, V: Src);
7472
7473 // Otherwise, f128 must live in FP128, so do a partwise move.
7474 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7475 &SystemZ::FP128BitRegClass);
7476
7477 SDValue Hi, Lo;
7478 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7479
7480 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7481 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7482
7483 SDNode *Pair = DAG.getMachineNode(
7484 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7485 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7486 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7487 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7488 return SDValue(Pair, 0);
7489}
7490
7491static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7492 const SDLoc &SL) {
7493 // If i128 is legal, just use a normal bitcast.
7494 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7495 return DAG.getBitcast(VT: MVT::i128, V: Src);
7496
7497 // Otherwise, f128 must live in FP128, so do a partwise move.
7498 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7499 &SystemZ::FP128BitRegClass);
7500
7501 SDValue LoFP =
7502 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7503 SDValue HiFP =
7504 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7505 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7506 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7507
7508 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7509}
7510
7511// Lower operations with invalid operand or result types.
7512void
7513SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7514 SmallVectorImpl<SDValue> &Results,
7515 SelectionDAG &DAG) const {
7516 switch (N->getOpcode()) {
7517 case ISD::ATOMIC_LOAD: {
7518 SDLoc DL(N);
7519 SDVTList Tys = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
7520 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7521 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7522 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7523 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7524
7525 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7526 if (N->getValueType(ResNo: 0) == MVT::f128)
7527 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7528 Results.push_back(Elt: Lowered);
7529 Results.push_back(Elt: Res.getValue(R: 1));
7530 break;
7531 }
7532 case ISD::ATOMIC_STORE: {
7533 SDLoc DL(N);
7534 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7535 SDValue Val = N->getOperand(Num: 1);
7536 if (Val.getValueType() == MVT::f128)
7537 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7538 Val = lowerI128ToGR128(DAG, In: Val);
7539
7540 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7541 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7542 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7543 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7544 // We have to enforce sequential consistency by performing a
7545 // serialization operation after the store.
7546 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7547 AtomicOrdering::SequentiallyConsistent)
7548 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7549 VT: MVT::Other, Op1: Res), 0);
7550 Results.push_back(Elt: Res);
7551 break;
7552 }
7553 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7554 SDLoc DL(N);
7555 SDVTList Tys = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::i32, VT3: MVT::Other);
7556 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7557 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7558 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7559 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7560 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7561 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7562 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7563 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7564 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7565 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7566 Results.push_back(Elt: Success);
7567 Results.push_back(Elt: Res.getValue(R: 2));
7568 break;
7569 }
7570 case ISD::BITCAST: {
7571 if (useSoftFloat())
7572 return;
7573 SDLoc DL(N);
7574 SDValue Src = N->getOperand(Num: 0);
7575 EVT SrcVT = Src.getValueType();
7576 EVT ResVT = N->getValueType(ResNo: 0);
7577 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7578 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7579 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7580 if (Subtarget.hasVector()) {
7581 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7582 Results.push_back(Elt: SDValue(
7583 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7584 } else {
7585 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7586 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7587 }
7588 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7589 SDValue ExtractedVal =
7590 Subtarget.hasVector()
7591 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7592 0)
7593 : convertFromF16(Op: Src, DL, DAG);
7594 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7595 }
7596 break;
7597 }
7598 case ISD::UINT_TO_FP:
7599 case ISD::SINT_TO_FP:
7600 case ISD::STRICT_UINT_TO_FP:
7601 case ISD::STRICT_SINT_TO_FP: {
7602 if (useSoftFloat())
7603 return;
7604 bool IsStrict = N->isStrictFPOpcode();
7605 SDLoc DL(N);
7606 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7607 EVT ResVT = N->getValueType(ResNo: 0);
7608 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7609 if (ResVT == MVT::f16) {
7610 if (!IsStrict) {
7611 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7612 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7613 } else {
7614 SDValue OpF32 =
7615 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7616 Ops: {Chain, InOp});
7617 SDValue F16Res;
7618 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7619 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7620 Results.push_back(Elt: F16Res);
7621 Results.push_back(Elt: Chain);
7622 }
7623 }
7624 break;
7625 }
7626 case ISD::FP_TO_UINT:
7627 case ISD::FP_TO_SINT:
7628 case ISD::STRICT_FP_TO_UINT:
7629 case ISD::STRICT_FP_TO_SINT: {
7630 if (useSoftFloat())
7631 return;
7632 bool IsStrict = N->isStrictFPOpcode();
7633 SDLoc DL(N);
7634 EVT ResVT = N->getValueType(ResNo: 0);
7635 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7636 EVT InVT = InOp->getValueType(ResNo: 0);
7637 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7638 if (InVT == MVT::f16) {
7639 if (!IsStrict) {
7640 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7641 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7642 } else {
7643 SDValue InF32;
7644 std::tie(args&: InF32, args&: Chain) =
7645 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7646 SDValue OpF32 =
7647 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7648 Ops: {Chain, InF32});
7649 Results.push_back(Elt: OpF32);
7650 Results.push_back(Elt: OpF32.getValue(R: 1));
7651 }
7652 }
7653 break;
7654 }
7655 case ISD::ATOMIC_SWAP:
7656 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7657 break;
7658 case ISD::ATOMIC_LOAD_ADD:
7659 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7660 break;
7661 case ISD::ATOMIC_LOAD_SUB:
7662 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
7663 break;
7664 case ISD::ATOMIC_LOAD_AND:
7665 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7666 break;
7667 case ISD::ATOMIC_LOAD_OR:
7668 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7669 break;
7670 case ISD::ATOMIC_LOAD_XOR:
7671 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7672 break;
7673 case ISD::ATOMIC_LOAD_NAND:
7674 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7675 break;
7676 case ISD::ATOMIC_LOAD_MIN:
7677 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7678 break;
7679 case ISD::ATOMIC_LOAD_MAX:
7680 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7681 break;
7682 case ISD::ATOMIC_LOAD_UMIN:
7683 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7684 break;
7685 case ISD::ATOMIC_LOAD_UMAX:
7686 lowerATOMIC_LOAD_OP(N, Results, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7687 break;
7688 default:
7689 llvm_unreachable("Unexpected node to lower");
7690 }
7691}
7692
7693void
7694SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7695 SmallVectorImpl<SDValue> &Results,
7696 SelectionDAG &DAG) const {
7697 return LowerOperationWrapper(N, Results, DAG);
7698}
7699
7700// Return true if VT is a vector whose elements are a whole number of bytes
7701// in width. Also check for presence of vector support.
7702bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7703 if (!Subtarget.hasVector())
7704 return false;
7705
7706 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7707}
7708
7709// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7710// producing a result of type ResVT. Op is a possibly bitcast version
7711// of the input vector and Index is the index (based on type VecVT) that
7712// should be extracted. Return the new extraction if a simplification
7713// was possible or if Force is true.
7714SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7715 EVT VecVT, SDValue Op,
7716 unsigned Index,
7717 DAGCombinerInfo &DCI,
7718 bool Force) const {
7719 SelectionDAG &DAG = DCI.DAG;
7720
7721 // The number of bytes being extracted.
7722 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7723
7724 for (;;) {
7725 unsigned Opcode = Op.getOpcode();
7726 if (Opcode == ISD::BITCAST)
7727 // Look through bitcasts.
7728 Op = Op.getOperand(i: 0);
7729 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7730 canTreatAsByteVector(VT: Op.getValueType())) {
7731 // Get a VPERM-like permute mask and see whether the bytes covered
7732 // by the extracted element are a contiguous sequence from one
7733 // source operand.
7734 SmallVector<int, SystemZ::VectorBytes> Bytes;
7735 if (!getVPermMask(ShuffleOp: Op, Bytes))
7736 break;
7737 int First;
7738 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7739 BytesPerElement, Base&: First))
7740 break;
7741 if (First < 0)
7742 return DAG.getUNDEF(VT: ResVT);
7743 // Make sure the contiguous sequence starts at a multiple of the
7744 // original element size.
7745 unsigned Byte = unsigned(First) % Bytes.size();
7746 if (Byte % BytesPerElement != 0)
7747 break;
7748 // We can get the extracted value directly from an input.
7749 Index = Byte / BytesPerElement;
7750 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7751 Force = true;
7752 } else if (Opcode == ISD::BUILD_VECTOR &&
7753 canTreatAsByteVector(VT: Op.getValueType())) {
7754 // We can only optimize this case if the BUILD_VECTOR elements are
7755 // at least as wide as the extracted value.
7756 EVT OpVT = Op.getValueType();
7757 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7758 if (OpBytesPerElement < BytesPerElement)
7759 break;
7760 // Make sure that the least-significant bit of the extracted value
7761 // is the least significant bit of an input.
7762 unsigned End = (Index + 1) * BytesPerElement;
7763 if (End % OpBytesPerElement != 0)
7764 break;
7765 // We're extracting the low part of one operand of the BUILD_VECTOR.
7766 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7767 EVT ResIntVT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7768 if (!isTypeLegal(VT: ResIntVT))
7769 break;
7770 if (!Op.getValueType().isInteger()) {
7771 EVT OpIntVT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7772 if (!isTypeLegal(VT: OpIntVT))
7773 break;
7774 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OpIntVT, Operand: Op);
7775 DCI.AddToWorklist(N: Op.getNode());
7776 }
7777 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResIntVT, Operand: Op);
7778 if (ResIntVT != ResVT) {
7779 DCI.AddToWorklist(N: Op.getNode());
7780 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7781 }
7782 return Op;
7783 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7784 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7785 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7786 canTreatAsByteVector(VT: Op.getValueType()) &&
7787 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7788 // Make sure that only the unextended bits are significant.
7789 EVT ExtVT = Op.getValueType();
7790 EVT OpVT = Op.getOperand(i: 0).getValueType();
7791 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7792 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7793 unsigned Byte = Index * BytesPerElement;
7794 unsigned SubByte = Byte % ExtBytesPerElement;
7795 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7796 if (SubByte < MinSubByte ||
7797 SubByte + BytesPerElement > ExtBytesPerElement)
7798 break;
7799 // Get the byte offset of the unextended element
7800 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7801 // ...then add the byte offset relative to that element.
7802 Byte += SubByte - MinSubByte;
7803 if (Byte % BytesPerElement != 0)
7804 break;
7805 Op = Op.getOperand(i: 0);
7806 Index = Byte / BytesPerElement;
7807 Force = true;
7808 } else
7809 break;
7810 }
7811 if (Force) {
7812 if (Op.getValueType() != VecVT) {
7813 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7814 DCI.AddToWorklist(N: Op.getNode());
7815 }
7816 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7817 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7818 }
7819 return SDValue();
7820}
7821
7822// Optimize vector operations in scalar value Op on the basis that Op
7823// is truncated to TruncVT.
7824SDValue SystemZTargetLowering::combineTruncateExtract(
7825 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7826 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7827 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7828 // of type TruncVT.
7829 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7830 TruncVT.getSizeInBits() % 8 == 0) {
7831 SDValue Vec = Op.getOperand(i: 0);
7832 EVT VecVT = Vec.getValueType();
7833 if (canTreatAsByteVector(VT: VecVT)) {
7834 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7835 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7836 unsigned TruncBytes = TruncVT.getStoreSize();
7837 if (BytesPerElement % TruncBytes == 0) {
7838 // Calculate the value of Y' in the above description. We are
7839 // splitting the original elements into Scale equal-sized pieces
7840 // and for truncation purposes want the last (least-significant)
7841 // of these pieces for IndexN. This is easiest to do by calculating
7842 // the start index of the following element and then subtracting 1.
7843 unsigned Scale = BytesPerElement / TruncBytes;
7844 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7845
7846 // Defer the creation of the bitcast from X to combineExtract,
7847 // which might be able to optimize the extraction.
7848 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7849 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7850 NumElements: VecVT.getStoreSize() / TruncBytes);
7851 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7852 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7853 }
7854 }
7855 }
7856 }
7857 return SDValue();
7858}
7859
7860SDValue SystemZTargetLowering::combineZERO_EXTEND(
7861 SDNode *N, DAGCombinerInfo &DCI) const {
7862 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7863 SelectionDAG &DAG = DCI.DAG;
7864 SDValue N0 = N->getOperand(Num: 0);
7865 EVT VT = N->getValueType(ResNo: 0);
7866 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7867 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7868 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7869 if (TrueOp && FalseOp) {
7870 SDLoc DL(N0);
7871 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7872 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7873 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7874 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7875 // If N0 has multiple uses, change other uses as well.
7876 if (!N0.hasOneUse()) {
7877 SDValue TruncSelect =
7878 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7879 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7880 }
7881 return NewSelect;
7882 }
7883 }
7884 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7885 // of the result is smaller than the size of X and all the truncated bits
7886 // of X are already zero.
7887 if (N0.getOpcode() == ISD::XOR &&
7888 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7889 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7890 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7891 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7892 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7893 KnownBits Known = DAG.computeKnownBits(Op: X);
7894 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7895 loBit: N0.getValueSizeInBits(),
7896 hiBit: VT.getSizeInBits());
7897 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7898 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7899 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7900 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7901 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7902 }
7903 }
7904 }
7905 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7906 // and VECTOR ADD COMPUTE CARRY for i128:
7907 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7908 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7909 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7910 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7911 // For vector types, these patterns are recognized in the .td file.
7912 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7913 N0.getOperand(i: 0).getValueType() == VT) {
7914 SDValue Op0 = N0.getOperand(i: 0);
7915 SDValue Op1 = N0.getOperand(i: 1);
7916 const ISD::CondCode CC = N0.getOperandCondCode(i: 2);
7917 switch (CC) {
7918 case ISD::SETULE:
7919 std::swap(a&: Op0, b&: Op1);
7920 [[fallthrough]];
7921 case ISD::SETUGE:
7922 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7923 case ISD::SETUGT:
7924 std::swap(a&: Op0, b&: Op1);
7925 [[fallthrough]];
7926 case ISD::SETULT:
7927 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7928 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7929 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7930 N2: Op0->getOperand(Num: 1));
7931 break;
7932 default:
7933 break;
7934 }
7935 }
7936
7937 return SDValue();
7938}
7939
7940SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7941 SDNode *N, DAGCombinerInfo &DCI) const {
7942 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7943 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7944 // into (select_cc LHS, RHS, -1, 0, COND)
7945 SelectionDAG &DAG = DCI.DAG;
7946 SDValue N0 = N->getOperand(Num: 0);
7947 EVT VT = N->getValueType(ResNo: 0);
7948 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7949 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7950 N0 = N0.getOperand(i: 0);
7951 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7952 SDLoc DL(N0);
7953 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7954 DAG.getAllOnesConstant(DL, VT),
7955 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7956 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7957 }
7958 return SDValue();
7959}
7960
7961SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7962 SDNode *N, DAGCombinerInfo &DCI) const {
7963 // Convert (sext (ashr (shl X, C1), C2)) to
7964 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7965 // cheap as narrower ones.
7966 SelectionDAG &DAG = DCI.DAG;
7967 SDValue N0 = N->getOperand(Num: 0);
7968 EVT VT = N->getValueType(ResNo: 0);
7969 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7970 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7971 SDValue Inner = N0.getOperand(i: 0);
7972 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7973 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7974 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7975 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7976 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7977 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7978 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7979 Operand: Inner.getOperand(i: 0));
7980 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7981 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7982 VT: ShiftVT));
7983 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7984 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7985 }
7986 }
7987 }
7988
7989 return SDValue();
7990}
7991
7992SDValue SystemZTargetLowering::combineMERGE(
7993 SDNode *N, DAGCombinerInfo &DCI) const {
7994 SelectionDAG &DAG = DCI.DAG;
7995 unsigned Opcode = N->getOpcode();
7996 SDValue Op0 = N->getOperand(Num: 0);
7997 SDValue Op1 = N->getOperand(Num: 1);
7998 if (Op0.getOpcode() == ISD::BITCAST)
7999 Op0 = Op0.getOperand(i: 0);
8000 if (ISD::isBuildVectorAllZeros(V: Op0)) {
8001 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
8002 // for v4f32.
8003 if (Op1 == N->getOperand(Num: 0))
8004 return Op1;
8005 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
8006 EVT VT = Op1.getValueType();
8007 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
8008 if (ElemBytes <= 4) {
8009 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
8010 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
8011 EVT InVT = VT.changeVectorElementTypeToInteger();
8012 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
8013 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
8014 if (VT != InVT) {
8015 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
8016 DCI.AddToWorklist(N: Op1.getNode());
8017 }
8018 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
8019 DCI.AddToWorklist(N: Op.getNode());
8020 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
8021 }
8022 }
8023 return SDValue();
8024}
8025
8026static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
8027 SDNode *&HiPart) {
8028 LoPart = HiPart = nullptr;
8029
8030 // Scan through all users.
8031 for (SDUse &Use : LD->uses()) {
8032 // Skip the uses of the chain.
8033 if (Use.getResNo() != 0)
8034 continue;
8035
8036 // Verify every user is a TRUNCATE to i64 of the low or high half.
8037 SDNode *User = Use.getUser();
8038 bool IsLoPart = true;
8039 if (User->getOpcode() == ISD::SRL &&
8040 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8041 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
8042 User = *User->user_begin();
8043 IsLoPart = false;
8044 }
8045 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
8046 return false;
8047
8048 if (IsLoPart) {
8049 if (LoPart)
8050 return false;
8051 LoPart = User;
8052 } else {
8053 if (HiPart)
8054 return false;
8055 HiPart = User;
8056 }
8057 }
8058 return true;
8059}
8060
8061static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
8062 SDNode *&HiPart) {
8063 LoPart = HiPart = nullptr;
8064
8065 // Scan through all users.
8066 for (SDUse &Use : LD->uses()) {
8067 // Skip the uses of the chain.
8068 if (Use.getResNo() != 0)
8069 continue;
8070
8071 // Verify every user is an EXTRACT_SUBREG of the low or high half.
8072 SDNode *User = Use.getUser();
8073 if (!User->hasOneUse() || !User->isMachineOpcode() ||
8074 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
8075 return false;
8076
8077 switch (User->getConstantOperandVal(Num: 1)) {
8078 case SystemZ::subreg_l64:
8079 if (LoPart)
8080 return false;
8081 LoPart = User;
8082 break;
8083 case SystemZ::subreg_h64:
8084 if (HiPart)
8085 return false;
8086 HiPart = User;
8087 break;
8088 default:
8089 return false;
8090 }
8091 }
8092 return true;
8093}
8094
8095SDValue SystemZTargetLowering::combineLOAD(
8096 SDNode *N, DAGCombinerInfo &DCI) const {
8097 SelectionDAG &DAG = DCI.DAG;
8098 EVT LdVT = N->getValueType(ResNo: 0);
8099 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
8100 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
8101 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8102 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
8103 if (PtrVT != LoadNodeVT) {
8104 SDLoc DL(LN);
8105 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
8106 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8107 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
8108 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
8109 MMO: LN->getMemOperand());
8110 }
8111 }
8112 }
8113 SDLoc DL(N);
8114
8115 // Replace a 128-bit load that is used solely to move its value into GPRs
8116 // by separate loads of both halves.
8117 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
8118 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
8119 SDNode *LoPart, *HiPart;
8120 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
8121 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
8122 // Rewrite each extraction as an independent load.
8123 SmallVector<SDValue, 2> ArgChains;
8124 if (HiPart) {
8125 SDValue EltLoad = DAG.getLoad(
8126 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
8127 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
8128 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8129
8130 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
8131 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8132 }
8133 if (LoPart) {
8134 SDValue EltLoad = DAG.getLoad(
8135 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
8136 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8137 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
8138 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8139
8140 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
8141 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8142 }
8143
8144 // Collect all chains via TokenFactor.
8145 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
8146 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
8147 DCI.AddToWorklist(N: Chain.getNode());
8148 return SDValue(N, 0);
8149 }
8150 }
8151
8152 if (LdVT.isVector() || LdVT.isInteger())
8153 return SDValue();
8154 // Transform a scalar load that is REPLICATEd as well as having other
8155 // use(s) to the form where the other use(s) use the first element of the
8156 // REPLICATE instead of the load. Otherwise instruction selection will not
8157 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
8158 // point loads.
8159
8160 SDValue Replicate;
8161 SmallVector<SDNode*, 8> OtherUses;
8162 for (SDUse &Use : N->uses()) {
8163 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8164 if (Replicate)
8165 return SDValue(); // Should never happen
8166 Replicate = SDValue(Use.getUser(), 0);
8167 } else if (Use.getResNo() == 0)
8168 OtherUses.push_back(Elt: Use.getUser());
8169 }
8170 if (!Replicate || OtherUses.empty())
8171 return SDValue();
8172
8173 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
8174 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
8175 // Update uses of the loaded Value while preserving old chains.
8176 for (SDNode *U : OtherUses) {
8177 SmallVector<SDValue, 8> Ops;
8178 for (SDValue Op : U->ops())
8179 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
8180 DAG.UpdateNodeOperands(N: U, Ops);
8181 }
8182 return SDValue(N, 0);
8183}
8184
8185bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8186 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8187 return true;
8188 if (Subtarget.hasVectorEnhancements2())
8189 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8190 return true;
8191 return false;
8192}
8193
8194static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8195 if (!VT.isVector() || !VT.isSimple() ||
8196 VT.getSizeInBits() != 128 ||
8197 VT.getScalarSizeInBits() % 8 != 0)
8198 return false;
8199
8200 unsigned NumElts = VT.getVectorNumElements();
8201 for (unsigned i = 0; i < NumElts; ++i) {
8202 if (M[i] < 0) continue; // ignore UNDEF indices
8203 if ((unsigned) M[i] != NumElts - 1 - i)
8204 return false;
8205 }
8206
8207 return true;
8208}
8209
8210static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8211 for (auto *U : StoredVal->users()) {
8212 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8213 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8214 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8215 continue;
8216 } else if (isa<BuildVectorSDNode>(Val: U)) {
8217 SDValue BuildVector = SDValue(U, 0);
8218 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8219 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8220 continue;
8221 }
8222 return false;
8223 }
8224 return true;
8225}
8226
8227static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8228 SDValue &HiPart) {
8229 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8230 return false;
8231
8232 SDValue Op0 = Val.getOperand(i: 0);
8233 SDValue Op1 = Val.getOperand(i: 1);
8234
8235 if (Op0.getOpcode() == ISD::SHL)
8236 std::swap(a&: Op0, b&: Op1);
8237 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8238 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8239 Op1.getConstantOperandVal(i: 1) != 64)
8240 return false;
8241 Op1 = Op1.getOperand(i: 0);
8242
8243 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8244 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8245 return false;
8246 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8247 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8248 return false;
8249
8250 LoPart = Op0.getOperand(i: 0);
8251 HiPart = Op1.getOperand(i: 0);
8252 return true;
8253}
8254
8255static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8256 SDValue &HiPart) {
8257 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8258 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8259 return false;
8260
8261 if (Val->getNumOperands() != 5 ||
8262 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8263 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8264 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8265 return false;
8266
8267 LoPart = Val->getOperand(Num: 1);
8268 HiPart = Val->getOperand(Num: 3);
8269 return true;
8270}
8271
8272SDValue SystemZTargetLowering::combineSTORE(
8273 SDNode *N, DAGCombinerInfo &DCI) const {
8274 SelectionDAG &DAG = DCI.DAG;
8275 auto *SN = cast<StoreSDNode>(Val: N);
8276 auto &Op1 = N->getOperand(Num: 1);
8277 EVT MemVT = SN->getMemoryVT();
8278
8279 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8280 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8281 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8282 if (PtrVT != StoreNodeVT) {
8283 SDLoc DL(SN);
8284 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8285 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8286 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8287 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8288 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8289 }
8290 }
8291
8292 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8293 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8294 // If X has wider elements then convert it to:
8295 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8296 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8297 if (SDValue Value =
8298 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8299 DCI.AddToWorklist(N: Value.getNode());
8300
8301 // Rewrite the store with the new form of stored value.
8302 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8303 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8304 MMO: SN->getMemOperand());
8305 }
8306 }
8307
8308 // combine STORE (LOAD_STACK_GUARD) into MOV_STACKGUARD_DAG
8309 if (Op1->isMachineOpcode() &&
8310 (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) {
8311 // Obtain the frame index the store was targeting.
8312 int FI = cast<FrameIndexSDNode>(Val: SN->getOperand(Num: 2))->getIndex();
8313 // Prepare operands of the MOV_STACKGUARD ISD Node - Chain and FrameIndex.
8314 SDValue Ops[] = {SN->getChain(), DAG.getTargetFrameIndex(FI, VT: MVT::i64)};
8315 return DAG.getNode(Opcode: SystemZISD::MOV_STACKGUARD, DL: SDLoc(SN), VT: MVT::Other, Ops);
8316 }
8317
8318 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8319 if (!SN->isTruncatingStore() &&
8320 Op1.getOpcode() == ISD::BSWAP &&
8321 Op1.getNode()->hasOneUse() &&
8322 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8323
8324 SDValue BSwapOp = Op1.getOperand(i: 0);
8325
8326 if (BSwapOp.getValueType() == MVT::i16)
8327 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8328
8329 SDValue Ops[] = {
8330 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8331 };
8332
8333 return
8334 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8335 Ops, MemVT, MMO: SN->getMemOperand());
8336 }
8337 // Combine STORE (element-swap) into VSTER
8338 if (!SN->isTruncatingStore() &&
8339 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8340 Op1.getNode()->hasOneUse() &&
8341 Subtarget.hasVectorEnhancements2()) {
8342 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8343 ArrayRef<int> ShuffleMask = SVN->getMask();
8344 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8345 SDValue Ops[] = {
8346 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8347 };
8348
8349 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8350 VTList: DAG.getVTList(VT: MVT::Other),
8351 Ops, MemVT, MMO: SN->getMemOperand());
8352 }
8353 }
8354
8355 // Combine STORE (READCYCLECOUNTER) into STCKF.
8356 if (!SN->isTruncatingStore() &&
8357 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8358 Op1.hasOneUse() &&
8359 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8360 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8361 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8362 VTList: DAG.getVTList(VT: MVT::Other),
8363 Ops, MemVT, MMO: SN->getMemOperand());
8364 }
8365
8366 // Transform a store of a 128-bit value moved from parts into two stores.
8367 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8368 SDValue LoPart, HiPart;
8369 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8370 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8371 SDLoc DL(SN);
8372 SDValue Chain0 = DAG.getStore(
8373 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8374 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8375 SDValue Chain1 = DAG.getStore(
8376 Chain: SN->getChain(), dl: DL, Val: LoPart,
8377 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8378 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8379 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8380
8381 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8382 }
8383 }
8384
8385 // Replicate a reg or immediate with VREP instead of scalar multiply or
8386 // immediate load. It seems best to do this during the first DAGCombine as
8387 // it is straight-forward to handle the zero-extend node in the initial
8388 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8389 // extracting an i16 element from a v16i8 vector).
8390 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8391 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8392 SDValue Word = SDValue();
8393 EVT WordVT;
8394
8395 // Find a replicated immediate and return it if found in Word and its
8396 // type in WordVT.
8397 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8398 // Some constants are better handled with a scalar store.
8399 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8400 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8401 return;
8402
8403 APInt Val = C->getAPIntValue();
8404 // Truncate Val in case of a truncating store.
8405 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8406 assert(SN->isTruncatingStore() &&
8407 "Non-truncating store and immediate value does not fit?");
8408 Val = Val.trunc(width: TotBytes * 8);
8409 }
8410
8411 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8412 if (VCI.isVectorConstantLegal(Subtarget) &&
8413 VCI.Opcode == SystemZISD::REPLICATE) {
8414 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8415 WordVT = VCI.VecVT.getScalarType();
8416 }
8417 };
8418
8419 // Find a replicated register and return it if found in Word and its type
8420 // in WordVT.
8421 auto FindReplicatedReg = [&](SDValue MulOp) {
8422 EVT MulVT = MulOp.getValueType();
8423 if (MulOp->getOpcode() == ISD::MUL &&
8424 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8425 // Find a zero extended value and its type.
8426 SDValue LHS = MulOp->getOperand(Num: 0);
8427 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8428 WordVT = LHS->getOperand(Num: 0).getValueType();
8429 else if (LHS->getOpcode() == ISD::AssertZext)
8430 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8431 else
8432 return;
8433 // Find a replicating constant, e.g. 0x00010001.
8434 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8435 SystemZVectorConstantInfo VCI(
8436 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8437 if (VCI.isVectorConstantLegal(Subtarget) &&
8438 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8439 WordVT == VCI.VecVT.getScalarType())
8440 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8441 }
8442 }
8443 };
8444
8445 if (isa<BuildVectorSDNode>(Val: Op1) &&
8446 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8447 SDValue SplatVal = Op1->getOperand(Num: 0);
8448 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8449 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8450 else
8451 FindReplicatedReg(SplatVal);
8452 } else {
8453 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8454 FindReplicatedImm(C, MemVT.getStoreSize());
8455 else
8456 FindReplicatedReg(Op1);
8457 }
8458
8459 if (Word != SDValue()) {
8460 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8461 "Bad type handling");
8462 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8463 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8464 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8465 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8466 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8467 }
8468 }
8469
8470 return SDValue();
8471}
8472
8473SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8474 SDNode *N, DAGCombinerInfo &DCI) const {
8475 SelectionDAG &DAG = DCI.DAG;
8476 // Combine element-swap (LOAD) into VLER
8477 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8478 N->getOperand(Num: 0).hasOneUse() &&
8479 Subtarget.hasVectorEnhancements2()) {
8480 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8481 ArrayRef<int> ShuffleMask = SVN->getMask();
8482 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8483 SDValue Load = N->getOperand(Num: 0);
8484 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8485
8486 // Create the element-swapping load.
8487 SDValue Ops[] = {
8488 LD->getChain(), // Chain
8489 LD->getBasePtr() // Ptr
8490 };
8491 SDValue ESLoad =
8492 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8493 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8494 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8495
8496 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8497 // by the load dead.
8498 DCI.CombineTo(N, Res: ESLoad);
8499
8500 // Next, combine the load away, we give it a bogus result value but a real
8501 // chain result. The result value is dead because the shuffle is dead.
8502 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8503
8504 // Return N so it doesn't get rechecked!
8505 return SDValue(N, 0);
8506 }
8507 }
8508
8509 return SDValue();
8510}
8511
8512SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8513 SDNode *N, DAGCombinerInfo &DCI) const {
8514 SelectionDAG &DAG = DCI.DAG;
8515
8516 if (!Subtarget.hasVector())
8517 return SDValue();
8518
8519 // Look through bitcasts that retain the number of vector elements.
8520 SDValue Op = N->getOperand(Num: 0);
8521 if (Op.getOpcode() == ISD::BITCAST &&
8522 Op.getValueType().isVector() &&
8523 Op.getOperand(i: 0).getValueType().isVector() &&
8524 Op.getValueType().getVectorNumElements() ==
8525 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8526 Op = Op.getOperand(i: 0);
8527
8528 // Pull BSWAP out of a vector extraction.
8529 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8530 EVT VecVT = Op.getValueType();
8531 EVT EltVT = VecVT.getVectorElementType();
8532 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8533 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8534 DCI.AddToWorklist(N: Op.getNode());
8535 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8536 if (EltVT != N->getValueType(ResNo: 0)) {
8537 DCI.AddToWorklist(N: Op.getNode());
8538 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8539 }
8540 return Op;
8541 }
8542
8543 // Try to simplify a vector extraction.
8544 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8545 SDValue Op0 = N->getOperand(Num: 0);
8546 EVT VecVT = Op0.getValueType();
8547 if (canTreatAsByteVector(VT: VecVT))
8548 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8549 Index: IndexN->getZExtValue(), DCI, Force: false);
8550 }
8551 return SDValue();
8552}
8553
8554SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8555 SDNode *N, DAGCombinerInfo &DCI) const {
8556 SelectionDAG &DAG = DCI.DAG;
8557 // (join_dwords X, X) == (replicate X)
8558 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8559 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8560 Operand: N->getOperand(Num: 0));
8561 return SDValue();
8562}
8563
8564static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8565 SDValue Chain1 = N1->getOperand(Num: 0);
8566 SDValue Chain2 = N2->getOperand(Num: 0);
8567
8568 // Trivial case: both nodes take the same chain.
8569 if (Chain1 == Chain2)
8570 return Chain1;
8571
8572 // FIXME - we could handle more complex cases via TokenFactor,
8573 // assuming we can verify that this would not create a cycle.
8574 return SDValue();
8575}
8576
8577SDValue SystemZTargetLowering::combineFP_ROUND(
8578 SDNode *N, DAGCombinerInfo &DCI) const {
8579
8580 if (!Subtarget.hasVector())
8581 return SDValue();
8582
8583 // (fpround (extract_vector_elt X 0))
8584 // (fpround (extract_vector_elt X 1)) ->
8585 // (extract_vector_elt (VROUND X) 0)
8586 // (extract_vector_elt (VROUND X) 2)
8587 //
8588 // This is a special case since the target doesn't really support v2f32s.
8589 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8590 SelectionDAG &DAG = DCI.DAG;
8591 SDValue Op0 = N->getOperand(Num: OpNo);
8592 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8593 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8594 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8595 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8596 Op0.getConstantOperandVal(i: 1) == 0) {
8597 SDValue Vec = Op0.getOperand(i: 0);
8598 for (auto *U : Vec->users()) {
8599 if (U != Op0.getNode() && U->hasOneUse() &&
8600 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8601 U->getOperand(Num: 0) == Vec &&
8602 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8603 U->getConstantOperandVal(Num: 1) == 1) {
8604 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8605 if (OtherRound.getOpcode() == N->getOpcode() &&
8606 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8607 OtherRound.getValueType() == MVT::f32) {
8608 SDValue VRound, Chain;
8609 if (N->isStrictFPOpcode()) {
8610 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8611 if (!Chain)
8612 continue;
8613 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8614 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8615 Chain = VRound.getValue(R: 1);
8616 } else
8617 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8618 VT: MVT::v4f32, Operand: Vec);
8619 DCI.AddToWorklist(N: VRound.getNode());
8620 SDValue Extract1 =
8621 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8622 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8623 DCI.AddToWorklist(N: Extract1.getNode());
8624 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8625 if (Chain)
8626 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8627 SDValue Extract0 =
8628 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8629 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8630 if (Chain)
8631 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8632 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8633 return Extract0;
8634 }
8635 }
8636 }
8637 }
8638 return SDValue();
8639}
8640
8641SDValue SystemZTargetLowering::combineFP_EXTEND(
8642 SDNode *N, DAGCombinerInfo &DCI) const {
8643
8644 if (!Subtarget.hasVector())
8645 return SDValue();
8646
8647 // (fpextend (extract_vector_elt X 0))
8648 // (fpextend (extract_vector_elt X 2)) ->
8649 // (extract_vector_elt (VEXTEND X) 0)
8650 // (extract_vector_elt (VEXTEND X) 1)
8651 //
8652 // This is a special case since the target doesn't really support v2f32s.
8653 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8654 SelectionDAG &DAG = DCI.DAG;
8655 SDValue Op0 = N->getOperand(Num: OpNo);
8656 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8657 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8658 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8659 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8660 Op0.getConstantOperandVal(i: 1) == 0) {
8661 SDValue Vec = Op0.getOperand(i: 0);
8662 for (auto *U : Vec->users()) {
8663 if (U != Op0.getNode() && U->hasOneUse() &&
8664 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8665 U->getOperand(Num: 0) == Vec &&
8666 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8667 U->getConstantOperandVal(Num: 1) == 2) {
8668 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8669 if (OtherExtend.getOpcode() == N->getOpcode() &&
8670 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8671 OtherExtend.getValueType() == MVT::f64) {
8672 SDValue VExtend, Chain;
8673 if (N->isStrictFPOpcode()) {
8674 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8675 if (!Chain)
8676 continue;
8677 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8678 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8679 Chain = VExtend.getValue(R: 1);
8680 } else
8681 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8682 VT: MVT::v2f64, Operand: Vec);
8683 DCI.AddToWorklist(N: VExtend.getNode());
8684 SDValue Extract1 =
8685 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8686 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8687 DCI.AddToWorklist(N: Extract1.getNode());
8688 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8689 if (Chain)
8690 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8691 SDValue Extract0 =
8692 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8693 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8694 if (Chain)
8695 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8696 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8697 return Extract0;
8698 }
8699 }
8700 }
8701 }
8702 return SDValue();
8703}
8704
8705SDValue SystemZTargetLowering::combineINT_TO_FP(
8706 SDNode *N, DAGCombinerInfo &DCI) const {
8707 if (DCI.Level != BeforeLegalizeTypes)
8708 return SDValue();
8709 SelectionDAG &DAG = DCI.DAG;
8710 LLVMContext &Ctx = *DAG.getContext();
8711 unsigned Opcode = N->getOpcode();
8712 EVT OutVT = N->getValueType(ResNo: 0);
8713 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8714 SDValue Op = N->getOperand(Num: 0);
8715 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8716 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8717
8718 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8719 // v2f64 = uint_to_fp v2i16
8720 // =>
8721 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8722 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8723 OutScalarBits <= 64) {
8724 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8725 EVT ExtVT = EVT::getVectorVT(
8726 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8727 unsigned ExtOpcode =
8728 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8729 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8730 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8731 }
8732 return SDValue();
8733}
8734
8735SDValue SystemZTargetLowering::combineFCOPYSIGN(
8736 SDNode *N, DAGCombinerInfo &DCI) const {
8737 SelectionDAG &DAG = DCI.DAG;
8738 EVT VT = N->getValueType(ResNo: 0);
8739 SDValue ValOp = N->getOperand(Num: 0);
8740 SDValue SignOp = N->getOperand(Num: 1);
8741
8742 // Remove the rounding which is not needed.
8743 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8744 SDValue WideOp = SignOp.getOperand(i: 0);
8745 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8746 }
8747
8748 return SDValue();
8749}
8750
8751SDValue SystemZTargetLowering::combineBSWAP(
8752 SDNode *N, DAGCombinerInfo &DCI) const {
8753 SelectionDAG &DAG = DCI.DAG;
8754 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8755 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8756 N->getOperand(Num: 0).hasOneUse() &&
8757 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8758 SDValue Load = N->getOperand(Num: 0);
8759 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8760
8761 // Create the byte-swapping load.
8762 SDValue Ops[] = {
8763 LD->getChain(), // Chain
8764 LD->getBasePtr() // Ptr
8765 };
8766 EVT LoadVT = N->getValueType(ResNo: 0);
8767 if (LoadVT == MVT::i16)
8768 LoadVT = MVT::i32;
8769 SDValue BSLoad =
8770 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8771 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8772 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8773
8774 // If this is an i16 load, insert the truncate.
8775 SDValue ResVal = BSLoad;
8776 if (N->getValueType(ResNo: 0) == MVT::i16)
8777 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8778
8779 // First, combine the bswap away. This makes the value produced by the
8780 // load dead.
8781 DCI.CombineTo(N, Res: ResVal);
8782
8783 // Next, combine the load away, we give it a bogus result value but a real
8784 // chain result. The result value is dead because the bswap is dead.
8785 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8786
8787 // Return N so it doesn't get rechecked!
8788 return SDValue(N, 0);
8789 }
8790
8791 // Look through bitcasts that retain the number of vector elements.
8792 SDValue Op = N->getOperand(Num: 0);
8793 if (Op.getOpcode() == ISD::BITCAST &&
8794 Op.getValueType().isVector() &&
8795 Op.getOperand(i: 0).getValueType().isVector() &&
8796 Op.getValueType().getVectorNumElements() ==
8797 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8798 Op = Op.getOperand(i: 0);
8799
8800 // Push BSWAP into a vector insertion if at least one side then simplifies.
8801 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8802 SDValue Vec = Op.getOperand(i: 0);
8803 SDValue Elt = Op.getOperand(i: 1);
8804 SDValue Idx = Op.getOperand(i: 2);
8805
8806 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8807 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8808 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8809 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8810 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8811 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8812 EVT VecVT = N->getValueType(ResNo: 0);
8813 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8814 if (VecVT != Vec.getValueType()) {
8815 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8816 DCI.AddToWorklist(N: Vec.getNode());
8817 }
8818 if (EltVT != Elt.getValueType()) {
8819 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8820 DCI.AddToWorklist(N: Elt.getNode());
8821 }
8822 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8823 DCI.AddToWorklist(N: Vec.getNode());
8824 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8825 DCI.AddToWorklist(N: Elt.getNode());
8826 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8827 N1: Vec, N2: Elt, N3: Idx);
8828 }
8829 }
8830
8831 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8832 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8833 if (SV && Op.hasOneUse()) {
8834 SDValue Op0 = Op.getOperand(i: 0);
8835 SDValue Op1 = Op.getOperand(i: 1);
8836
8837 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8838 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8839 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8840 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8841 EVT VecVT = N->getValueType(ResNo: 0);
8842 if (VecVT != Op0.getValueType()) {
8843 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8844 DCI.AddToWorklist(N: Op0.getNode());
8845 }
8846 if (VecVT != Op1.getValueType()) {
8847 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8848 DCI.AddToWorklist(N: Op1.getNode());
8849 }
8850 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8851 DCI.AddToWorklist(N: Op0.getNode());
8852 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8853 DCI.AddToWorklist(N: Op1.getNode());
8854 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8855 }
8856 }
8857
8858 return SDValue();
8859}
8860
8861SDValue SystemZTargetLowering::combineSETCC(
8862 SDNode *N, DAGCombinerInfo &DCI) const {
8863 SelectionDAG &DAG = DCI.DAG;
8864 const ISD::CondCode CC = N->getOperandCondCode(Num: 2);
8865 const SDValue LHS = N->getOperand(Num: 0);
8866 const SDValue RHS = N->getOperand(Num: 1);
8867 bool CmpNull = isNullConstant(V: RHS);
8868 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8869 EVT VT = N->getValueType(ResNo: 0);
8870 SDLoc DL(N);
8871
8872 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8873 // change the outer compare to a i128 compare. This will normally
8874 // allow the reduction to be recognized in adjustICmp128, and even if
8875 // not, the i128 compare will still generate better code.
8876 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8877 SDValue Src = peekThroughBitcasts(V: LHS);
8878 if (Src.getOpcode() == ISD::SETCC &&
8879 Src.getValueType().isFixedLengthVector() &&
8880 Src.getValueType().getScalarType() == MVT::i1) {
8881 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8882 if (CmpVT.getSizeInBits() == 128) {
8883 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8884 SDValue LHS =
8885 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8886 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8887 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8888 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8889 Flags: N->getFlags());
8890 }
8891 }
8892 }
8893
8894 return SDValue();
8895}
8896
8897static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8898 unsigned Depth = 0) {
8899 // Limit depth of potentially exponential walk.
8900 if (Depth > 5)
8901 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8902
8903 switch (Val.getOpcode()) {
8904 default:
8905 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8906 case SystemZISD::IPM:
8907 if (Val.getOperand(i: 0).getOpcode() == SystemZISD::CLC ||
8908 Val.getOperand(i: 0).getOpcode() == SystemZISD::STRCMP)
8909 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ICMP);
8910 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ANY);
8911 case SystemZISD::SELECT_CCMASK: {
8912 SDValue Op4CCReg = Val.getOperand(i: 4);
8913 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8914 Op4CCReg.getOpcode() == SystemZISD::TM) {
8915 auto [OpCC, OpCCValid] = findCCUse(Val: Op4CCReg.getOperand(i: 0), Depth: Depth + 1);
8916 if (OpCC != SDValue())
8917 return std::make_pair(x&: OpCC, y&: OpCCValid);
8918 }
8919 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8920 if (!CCValid)
8921 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8922 int CCValidVal = CCValid->getZExtValue();
8923 return std::make_pair(x&: Op4CCReg, y&: CCValidVal);
8924 }
8925 case ISD::ADD:
8926 case ISD::AND:
8927 case ISD::OR:
8928 case ISD::XOR:
8929 case ISD::SHL:
8930 case ISD::SRA:
8931 case ISD::SRL:
8932 auto [Op0CC, Op0CCValid] = findCCUse(Val: Val.getOperand(i: 0), Depth: Depth + 1);
8933 if (Op0CC != SDValue())
8934 return std::make_pair(x&: Op0CC, y&: Op0CCValid);
8935 return findCCUse(Val: Val.getOperand(i: 1), Depth: Depth + 1);
8936 }
8937}
8938
8939static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8940 SelectionDAG &DAG);
8941
8942SmallVector<SDValue, 4> static simplifyAssumingCCVal(SDValue &Val, SDValue &CC,
8943 SelectionDAG &DAG) {
8944 SDLoc DL(Val);
8945 auto Opcode = Val.getOpcode();
8946 switch (Opcode) {
8947 default:
8948 return {};
8949 case ISD::Constant:
8950 return {Val, Val, Val, Val};
8951 case SystemZISD::IPM: {
8952 SDValue IPMOp0 = Val.getOperand(i: 0);
8953 if (IPMOp0 != CC)
8954 return {};
8955 SmallVector<SDValue, 4> ShiftedCCVals;
8956 for (auto CC : {0, 1, 2, 3})
8957 ShiftedCCVals.emplace_back(
8958 Args: DAG.getConstant(Val: (CC << SystemZ::IPM_CC), DL, VT: MVT::i32));
8959 return ShiftedCCVals;
8960 }
8961 case SystemZISD::SELECT_CCMASK: {
8962 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8963 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8964 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8965 if (!CCValid || !CCMask)
8966 return {};
8967
8968 int CCValidVal = CCValid->getZExtValue();
8969 int CCMaskVal = CCMask->getZExtValue();
8970 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8971 // recursive call to simplifyAssumingCCVal.
8972 SDValue Op4CCReg = Val.getOperand(i: 4);
8973 if (Op4CCReg != CC)
8974 combineCCMask(CCReg&: Op4CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8975 if (Op4CCReg != CC)
8976 return {};
8977 const auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC, DAG);
8978 const auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC, DAG);
8979 if (TrueSDVals.empty() || FalseSDVals.empty())
8980 return {};
8981 SmallVector<SDValue, 4> MergedSDVals;
8982 for (auto &CCVal : {0, 1, 2, 3})
8983 MergedSDVals.emplace_back(Args: ((CCMaskVal & (1 << (3 - CCVal))) != 0)
8984 ? TrueSDVals[CCVal]
8985 : FalseSDVals[CCVal]);
8986 return MergedSDVals;
8987 }
8988 case ISD::ADD:
8989 case ISD::AND:
8990 case ISD::OR:
8991 case ISD::XOR:
8992 case ISD::SRA:
8993 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8994 // would clobber CC).
8995 if (!Val.hasOneUse())
8996 return {};
8997 [[fallthrough]];
8998 case ISD::SHL:
8999 case ISD::SRL:
9000 SDValue Op0 = Val.getOperand(i: 0), Op1 = Val.getOperand(i: 1);
9001 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC, DAG);
9002 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC, DAG);
9003 if (Op0SDVals.empty() || Op1SDVals.empty())
9004 return {};
9005 SmallVector<SDValue, 4> BinaryOpSDVals;
9006 for (auto CCVal : {0, 1, 2, 3})
9007 BinaryOpSDVals.emplace_back(Args: DAG.getNode(
9008 Opcode, DL, VT: Val.getValueType(), N1: Op0SDVals[CCVal], N2: Op1SDVals[CCVal]));
9009 return BinaryOpSDVals;
9010 }
9011}
9012
9013static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
9014 SelectionDAG &DAG) {
9015 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
9016 // set by the CCReg instruction using the CCValid / CCMask masks,
9017 // If the CCReg instruction is itself a ICMP / TM testing the condition
9018 // code set by some other instruction, see whether we can directly
9019 // use that condition code.
9020 auto *CCNode = CCReg.getNode();
9021 if (!CCNode)
9022 return false;
9023
9024 if (CCNode->getOpcode() == SystemZISD::TM) {
9025 if (CCValid != SystemZ::CCMASK_TM)
9026 return false;
9027 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
9028 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
9029 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
9030 if (!Op0Node || !Op1Node)
9031 return -1;
9032 auto Op0APVal = Op0Node->getAPIntValue();
9033 auto Op1APVal = Op1Node->getAPIntValue();
9034 auto Result = Op0APVal & Op1APVal;
9035 bool AllOnes = Result == Op1APVal;
9036 bool AllZeros = Result == 0;
9037 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
9038 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
9039 };
9040 SDValue Op0 = CCNode->getOperand(Num: 0);
9041 SDValue Op1 = CCNode->getOperand(Num: 1);
9042 auto [Op0CC, Op0CCValid] = findCCUse(Val: Op0);
9043 if (Op0CC == SDValue())
9044 return false;
9045 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC&: Op0CC, DAG);
9046 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC&: Op0CC, DAG);
9047 if (Op0SDVals.empty() || Op1SDVals.empty())
9048 return false;
9049 int NewCCMask = 0;
9050 for (auto CC : {0, 1, 2, 3}) {
9051 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
9052 if (CCVal < 0)
9053 return false;
9054 NewCCMask <<= 1;
9055 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9056 }
9057 NewCCMask &= Op0CCValid;
9058 CCReg = Op0CC;
9059 CCMask = NewCCMask;
9060 CCValid = Op0CCValid;
9061 return true;
9062 }
9063 if (CCNode->getOpcode() != SystemZISD::ICMP ||
9064 CCValid != SystemZ::CCMASK_ICMP)
9065 return false;
9066
9067 SDValue CmpOp0 = CCNode->getOperand(Num: 0);
9068 SDValue CmpOp1 = CCNode->getOperand(Num: 1);
9069 SDValue CmpOp2 = CCNode->getOperand(Num: 2);
9070 auto [Op0CC, Op0CCValid] = findCCUse(Val: CmpOp0);
9071 if (Op0CC != SDValue()) {
9072 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: CmpOp0, CC&: Op0CC, DAG);
9073 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: CmpOp1, CC&: Op0CC, DAG);
9074 if (Op0SDVals.empty() || Op1SDVals.empty())
9075 return false;
9076
9077 auto *CmpType = dyn_cast<ConstantSDNode>(Val&: CmpOp2);
9078 auto CmpTypeVal = CmpType->getZExtValue();
9079 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
9080 const SDValue &Op1Val) {
9081 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
9082 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
9083 if (!Op0Node || !Op1Node)
9084 return -1;
9085 auto Op0APVal = Op0Node->getAPIntValue();
9086 auto Op1APVal = Op1Node->getAPIntValue();
9087 if (CmpTypeVal == SystemZICMP::SignedOnly)
9088 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(RHS: Op1APVal) ? 1 : 2;
9089 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(RHS: Op1APVal) ? 1 : 2;
9090 };
9091 int NewCCMask = 0;
9092 for (auto CC : {0, 1, 2, 3}) {
9093 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
9094 if (CCVal < 0)
9095 return false;
9096 NewCCMask <<= 1;
9097 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9098 }
9099 NewCCMask &= Op0CCValid;
9100 CCMask = NewCCMask;
9101 CCReg = Op0CC;
9102 CCValid = Op0CCValid;
9103 return true;
9104 }
9105
9106 return false;
9107}
9108
9109// Merging versus split in multiple branches cost.
9110TargetLoweringBase::CondMergingParams
9111SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
9112 const Value *Lhs,
9113 const Value *Rhs,
9114 const Function *) const {
9115 const auto isFlagOutOpCC = [](const Value *V) {
9116 using namespace llvm::PatternMatch;
9117 const Value *RHSVal;
9118 const APInt *RHSC;
9119 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
9120 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
9121 if (match(V: I->getOperand(i: 0), P: m_And(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC))) ||
9122 match(V: I, P: m_Cmp(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC)))) {
9123 if (const auto *CB = dyn_cast<CallBase>(Val: RHSVal)) {
9124 if (CB->isInlineAsm()) {
9125 const InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
9126 return IA && IA->getConstraintString().contains(Other: "{@cc}");
9127 }
9128 }
9129 }
9130 }
9131 return false;
9132 };
9133 // Pattern (ICmp %asm) or (ICmp (And %asm)).
9134 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
9135 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
9136 // conditionals will be merged or else conditionals will be split.
9137 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
9138 return {.BaseCost: 3, .LikelyBias: 0, .UnlikelyBias: -1};
9139 // Default.
9140 return {.BaseCost: -1, .LikelyBias: -1, .UnlikelyBias: -1};
9141}
9142
9143SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
9144 DAGCombinerInfo &DCI) const {
9145 SelectionDAG &DAG = DCI.DAG;
9146
9147 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
9148 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9149 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9150 if (!CCValid || !CCMask)
9151 return SDValue();
9152
9153 int CCValidVal = CCValid->getZExtValue();
9154 int CCMaskVal = CCMask->getZExtValue();
9155 SDValue Chain = N->getOperand(Num: 0);
9156 SDValue CCReg = N->getOperand(Num: 4);
9157 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
9158 // the modified BR_CCMASK with the new values.
9159 // In order to avoid conditional branches with full or empty cc masks, do not
9160 // do this if ccmask is 0 or equal to ccvalid.
9161 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG) && CCMaskVal != 0 &&
9162 CCMaskVal != CCValidVal)
9163 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
9164 N1: Chain,
9165 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9166 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
9167 N4: N->getOperand(Num: 3), N5: CCReg);
9168 return SDValue();
9169}
9170
9171SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9172 SDNode *N, DAGCombinerInfo &DCI) const {
9173 SelectionDAG &DAG = DCI.DAG;
9174
9175 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
9176 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9177 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
9178 if (!CCValid || !CCMask)
9179 return SDValue();
9180
9181 int CCValidVal = CCValid->getZExtValue();
9182 int CCMaskVal = CCMask->getZExtValue();
9183 SDValue CCReg = N->getOperand(Num: 4);
9184
9185 bool IsCombinedCCReg = combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
9186
9187 // Populate SDVals vector for each condition code ccval for given Val, which
9188 // can again be another nested select_ccmask with the same CC.
9189 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
9190 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9191 SmallVector<SDValue, 4> Res;
9192 if (Val.getOperand(i: 4) != CCReg)
9193 return SmallVector<SDValue, 4>{};
9194 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
9195 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
9196 if (!CCMask)
9197 return SmallVector<SDValue, 4>{};
9198
9199 int CCMaskVal = CCMask->getZExtValue();
9200 for (auto &CC : {0, 1, 2, 3})
9201 Res.emplace_back(Args&: ((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
9202 : FalseVal);
9203 return Res;
9204 }
9205 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9206 };
9207 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9208 // with CCReg found by combineCCMask or original CCReg.
9209 SDValue TrueVal = N->getOperand(Num: 0);
9210 SDValue FalseVal = N->getOperand(Num: 1);
9211 auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC&: CCReg, DAG);
9212 auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC&: CCReg, DAG);
9213 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9214 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9215 if (TrueSDVals.empty())
9216 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9217 if (FalseSDVals.empty())
9218 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9219 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9220 SmallSet<SDValue, 4> MergedSDValsSet;
9221 // Ignoring CC values outside CCValiid.
9222 for (auto CC : {0, 1, 2, 3}) {
9223 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9224 MergedSDValsSet.insert(V: ((CCMaskVal & (1 << (3 - CC))) != 0)
9225 ? TrueSDVals[CC]
9226 : FalseSDVals[CC]);
9227 }
9228 if (MergedSDValsSet.size() == 1)
9229 return *MergedSDValsSet.begin();
9230 if (MergedSDValsSet.size() == 2) {
9231 auto BeginIt = MergedSDValsSet.begin();
9232 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(x: BeginIt);
9233 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9234 std::swap(a&: NewTrueVal, b&: NewFalseVal);
9235 int NewCCMask = 0;
9236 for (auto CC : {0, 1, 2, 3}) {
9237 NewCCMask <<= 1;
9238 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9239 ? (TrueSDVals[CC] == NewTrueVal)
9240 : (FalseSDVals[CC] == NewTrueVal);
9241 }
9242 CCMaskVal = NewCCMask;
9243 CCMaskVal &= CCValidVal;
9244 TrueVal = NewTrueVal;
9245 FalseVal = NewFalseVal;
9246 IsCombinedCCReg = true;
9247 }
9248 }
9249 // If the condition is trivially false or trivially true after
9250 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9251 // (possibly modified by constructCCSDValsFromSELECT).
9252 if (CCMaskVal == 0)
9253 return FalseVal;
9254 if (CCMaskVal == CCValidVal)
9255 return TrueVal;
9256
9257 if (IsCombinedCCReg)
9258 return DAG.getNode(
9259 Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: TrueVal,
9260 N2: FalseVal, N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9261 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32), N5: CCReg);
9262
9263 return SDValue();
9264}
9265
9266SDValue SystemZTargetLowering::combineGET_CCMASK(
9267 SDNode *N, DAGCombinerInfo &DCI) const {
9268
9269 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9270 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9271 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9272 if (!CCValid || !CCMask)
9273 return SDValue();
9274 int CCValidVal = CCValid->getZExtValue();
9275 int CCMaskVal = CCMask->getZExtValue();
9276
9277 SDValue Select = N->getOperand(Num: 0);
9278 if (Select->getOpcode() == ISD::TRUNCATE)
9279 Select = Select->getOperand(Num: 0);
9280 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9281 return SDValue();
9282
9283 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
9284 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
9285 if (!SelectCCValid || !SelectCCMask)
9286 return SDValue();
9287 int SelectCCValidVal = SelectCCValid->getZExtValue();
9288 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9289
9290 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
9291 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
9292 if (!TrueVal || !FalseVal)
9293 return SDValue();
9294 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9295 ;
9296 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9297 SelectCCMaskVal ^= SelectCCValidVal;
9298 else
9299 return SDValue();
9300
9301 if (SelectCCValidVal & ~CCValidVal)
9302 return SDValue();
9303 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9304 return SDValue();
9305
9306 return Select->getOperand(Num: 4);
9307}
9308
9309SDValue SystemZTargetLowering::combineIntDIVREM(
9310 SDNode *N, DAGCombinerInfo &DCI) const {
9311 SelectionDAG &DAG = DCI.DAG;
9312 EVT VT = N->getValueType(ResNo: 0);
9313 // In the case where the divisor is a vector of constants a cheaper
9314 // sequence of instructions can replace the divide. BuildSDIV is called to
9315 // do this during DAG combining, but it only succeeds when it can build a
9316 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9317 // since it is not Legal but Custom it can only happen before
9318 // legalization. Therefore we must scalarize this early before Combine
9319 // 1. For widened vectors, this is already the result of type legalization.
9320 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9321 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
9322 return DAG.UnrollVectorOp(N);
9323 return SDValue();
9324}
9325
9326
9327// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9328// This is closely modeled after the common-code combineShiftToMULH.
9329SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9330 SDNode *N, DAGCombinerInfo &DCI) const {
9331 SelectionDAG &DAG = DCI.DAG;
9332 SDLoc DL(N);
9333
9334 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9335 "SRL or SRA node is required here!");
9336
9337 if (!Subtarget.hasVector())
9338 return SDValue();
9339
9340 // Check the shift amount. Proceed with the transformation if the shift
9341 // amount is constant.
9342 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
9343 if (!ShiftAmtSrc)
9344 return SDValue();
9345
9346 // The operation feeding into the shift must be an add.
9347 SDValue ShiftOperand = N->getOperand(Num: 0);
9348 if (ShiftOperand.getOpcode() != ISD::ADD)
9349 return SDValue();
9350
9351 // One operand of the add must be a multiply.
9352 SDValue MulOp = ShiftOperand.getOperand(i: 0);
9353 SDValue AddOp = ShiftOperand.getOperand(i: 1);
9354 if (MulOp.getOpcode() != ISD::MUL) {
9355 if (AddOp.getOpcode() != ISD::MUL)
9356 return SDValue();
9357 std::swap(a&: MulOp, b&: AddOp);
9358 }
9359
9360 // All operands must be equivalent extend nodes.
9361 SDValue LeftOp = MulOp.getOperand(i: 0);
9362 SDValue RightOp = MulOp.getOperand(i: 1);
9363
9364 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9365 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9366
9367 if (!IsSignExt && !IsZeroExt)
9368 return SDValue();
9369
9370 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
9371 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9372
9373 SDValue MulhRightOp;
9374 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
9375 unsigned ActiveBits = IsSignExt
9376 ? Constant->getAPIntValue().getSignificantBits()
9377 : Constant->getAPIntValue().getActiveBits();
9378 if (ActiveBits > NarrowVTSize)
9379 return SDValue();
9380 MulhRightOp = DAG.getConstant(
9381 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9382 VT: NarrowVT);
9383 } else {
9384 if (LeftOp.getOpcode() != RightOp.getOpcode())
9385 return SDValue();
9386 // Check that the two extend nodes are the same type.
9387 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
9388 return SDValue();
9389 MulhRightOp = RightOp.getOperand(i: 0);
9390 }
9391
9392 SDValue MulhAddOp;
9393 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
9394 unsigned ActiveBits = IsSignExt
9395 ? Constant->getAPIntValue().getSignificantBits()
9396 : Constant->getAPIntValue().getActiveBits();
9397 if (ActiveBits > NarrowVTSize)
9398 return SDValue();
9399 MulhAddOp = DAG.getConstant(
9400 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9401 VT: NarrowVT);
9402 } else {
9403 if (LeftOp.getOpcode() != AddOp.getOpcode())
9404 return SDValue();
9405 // Check that the two extend nodes are the same type.
9406 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
9407 return SDValue();
9408 MulhAddOp = AddOp.getOperand(i: 0);
9409 }
9410
9411 EVT WideVT = LeftOp.getValueType();
9412 // Proceed with the transformation if the wide types match.
9413 assert((WideVT == RightOp.getValueType()) &&
9414 "Cannot have a multiply node with two different operand types.");
9415 assert((WideVT == AddOp.getValueType()) &&
9416 "Cannot have an add node with two different operand types.");
9417
9418 // Proceed with the transformation if the wide type is twice as large
9419 // as the narrow type.
9420 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9421 return SDValue();
9422
9423 // Check the shift amount with the narrow type size.
9424 // Proceed with the transformation if the shift amount is the width
9425 // of the narrow type.
9426 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9427 if (ShiftAmt != NarrowVTSize)
9428 return SDValue();
9429
9430 // Proceed if we support the multiply-and-add-high operation.
9431 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9432 NarrowVT == MVT::v4i32 ||
9433 (Subtarget.hasVectorEnhancements3() &&
9434 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9435 return SDValue();
9436
9437 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9438 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9439 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9440 N2: MulhRightOp, N3: MulhAddOp);
9441 bool IsSigned = N->getOpcode() == ISD::SRA;
9442 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9443}
9444
9445// Op is an operand of a multiplication. Check whether this can be folded
9446// into an even/odd widening operation; if so, return the opcode to be used
9447// and update Op to the appropriate sub-operand. Note that the caller must
9448// verify that *both* operands of the multiplication support the operation.
9449static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9450 const SystemZSubtarget &Subtarget,
9451 SDValue &Op) {
9452 EVT VT = Op.getValueType();
9453
9454 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9455 // to selecting the even or odd vector elements.
9456 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9457 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9458 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9459 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9460 unsigned NumElts = VT.getVectorNumElements();
9461 Op = Op.getOperand(i: 0);
9462 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9463 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9464 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9465 ArrayRef<int> ShuffleMask = SVN->getMask();
9466 bool CanUseEven = true, CanUseOdd = true;
9467 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9468 if (ShuffleMask[Elt] == -1)
9469 continue;
9470 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9471 CanUseEven = false;
9472 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9473 CanUseOdd = false;
9474 }
9475 Op = Op.getOperand(i: 0);
9476 if (CanUseEven)
9477 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9478 if (CanUseOdd)
9479 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9480 }
9481 }
9482
9483 // For z17, we can also support the v2i64->i128 case, which looks like
9484 // (sign/zero_extend (extract_vector_elt X 0/1))
9485 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9486 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9487 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9488 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9489 Op = Op.getOperand(i: 0);
9490 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9491 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9492 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9493 unsigned Elem = Op.getConstantOperandVal(i: 1);
9494 Op = Op.getOperand(i: 0);
9495 if (Elem == 0)
9496 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9497 if (Elem == 1)
9498 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9499 }
9500 }
9501
9502 return 0;
9503}
9504
9505SDValue SystemZTargetLowering::combineMUL(
9506 SDNode *N, DAGCombinerInfo &DCI) const {
9507 SelectionDAG &DAG = DCI.DAG;
9508
9509 // Detect even/odd widening multiplication.
9510 SDValue Op0 = N->getOperand(Num: 0);
9511 SDValue Op1 = N->getOperand(Num: 1);
9512 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9513 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9514 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9515 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9516
9517 return SDValue();
9518}
9519
9520SDValue SystemZTargetLowering::combineINTRINSIC(
9521 SDNode *N, DAGCombinerInfo &DCI) const {
9522 SelectionDAG &DAG = DCI.DAG;
9523
9524 unsigned Id = N->getConstantOperandVal(Num: 1);
9525 switch (Id) {
9526 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9527 // or larger is simply a vector load.
9528 case Intrinsic::s390_vll:
9529 case Intrinsic::s390_vlrl:
9530 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9531 if (C->getZExtValue() >= 15)
9532 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9533 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9534 break;
9535 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9536 case Intrinsic::s390_vstl:
9537 case Intrinsic::s390_vstrl:
9538 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9539 if (C->getZExtValue() >= 15)
9540 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9541 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9542 break;
9543 }
9544
9545 return SDValue();
9546}
9547
9548SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9549 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9550 return N->getOperand(Num: 0);
9551 return N;
9552}
9553
9554SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9555 DAGCombinerInfo &DCI) const {
9556 switch(N->getOpcode()) {
9557 default: break;
9558 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9559 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9560 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9561 case SystemZISD::MERGE_HIGH:
9562 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9563 case ISD::LOAD: return combineLOAD(N, DCI);
9564 case ISD::STORE: return combineSTORE(N, DCI);
9565 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9566 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9567 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9568 case ISD::STRICT_FP_ROUND:
9569 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9570 case ISD::STRICT_FP_EXTEND:
9571 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9572 case ISD::SINT_TO_FP:
9573 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9574 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9575 case ISD::BSWAP: return combineBSWAP(N, DCI);
9576 case ISD::SETCC: return combineSETCC(N, DCI);
9577 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9578 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9579 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9580 case ISD::SRL:
9581 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9582 case ISD::MUL: return combineMUL(N, DCI);
9583 case ISD::SDIV:
9584 case ISD::UDIV:
9585 case ISD::SREM:
9586 case ISD::UREM: return combineIntDIVREM(N, DCI);
9587 case ISD::INTRINSIC_W_CHAIN:
9588 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9589 }
9590
9591 return SDValue();
9592}
9593
9594// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9595// are for Op.
9596static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9597 unsigned OpNo) {
9598 EVT VT = Op.getValueType();
9599 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9600 APInt SrcDemE;
9601 unsigned Opcode = Op.getOpcode();
9602 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9603 unsigned Id = Op.getConstantOperandVal(i: 0);
9604 switch (Id) {
9605 case Intrinsic::s390_vpksh: // PACKS
9606 case Intrinsic::s390_vpksf:
9607 case Intrinsic::s390_vpksg:
9608 case Intrinsic::s390_vpkshs: // PACKS_CC
9609 case Intrinsic::s390_vpksfs:
9610 case Intrinsic::s390_vpksgs:
9611 case Intrinsic::s390_vpklsh: // PACKLS
9612 case Intrinsic::s390_vpklsf:
9613 case Intrinsic::s390_vpklsg:
9614 case Intrinsic::s390_vpklshs: // PACKLS_CC
9615 case Intrinsic::s390_vpklsfs:
9616 case Intrinsic::s390_vpklsgs:
9617 // VECTOR PACK truncates the elements of two source vectors into one.
9618 SrcDemE = DemandedElts;
9619 if (OpNo == 2)
9620 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9621 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9622 break;
9623 // VECTOR UNPACK extends half the elements of the source vector.
9624 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9625 case Intrinsic::s390_vuphh:
9626 case Intrinsic::s390_vuphf:
9627 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9628 case Intrinsic::s390_vuplhh:
9629 case Intrinsic::s390_vuplhf:
9630 SrcDemE = APInt(NumElts * 2, 0);
9631 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9632 break;
9633 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9634 case Intrinsic::s390_vuplhw:
9635 case Intrinsic::s390_vuplf:
9636 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9637 case Intrinsic::s390_vupllh:
9638 case Intrinsic::s390_vupllf:
9639 SrcDemE = APInt(NumElts * 2, 0);
9640 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9641 break;
9642 case Intrinsic::s390_vpdi: {
9643 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9644 SrcDemE = APInt(NumElts, 0);
9645 if (!DemandedElts[OpNo - 1])
9646 break;
9647 unsigned Mask = Op.getConstantOperandVal(i: 3);
9648 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9649 // Demand input element 0 or 1, given by the mask bit value.
9650 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9651 break;
9652 }
9653 case Intrinsic::s390_vsldb: {
9654 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9655 assert(VT == MVT::v16i8 && "Unexpected type.");
9656 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9657 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9658 unsigned NumSrc0Els = 16 - FirstIdx;
9659 SrcDemE = APInt(NumElts, 0);
9660 if (OpNo == 1) {
9661 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9662 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9663 } else {
9664 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9665 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9666 }
9667 break;
9668 }
9669 case Intrinsic::s390_vperm:
9670 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9671 break;
9672 default:
9673 llvm_unreachable("Unhandled intrinsic.");
9674 break;
9675 }
9676 } else {
9677 switch (Opcode) {
9678 case SystemZISD::JOIN_DWORDS:
9679 // Scalar operand.
9680 SrcDemE = APInt(1, 1);
9681 break;
9682 case SystemZISD::SELECT_CCMASK:
9683 SrcDemE = DemandedElts;
9684 break;
9685 default:
9686 llvm_unreachable("Unhandled opcode.");
9687 break;
9688 }
9689 }
9690 return SrcDemE;
9691}
9692
9693static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9694 const APInt &DemandedElts,
9695 const SelectionDAG &DAG, unsigned Depth,
9696 unsigned OpNo) {
9697 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9698 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9699 KnownBits LHSKnown =
9700 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9701 KnownBits RHSKnown =
9702 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9703 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9704}
9705
9706void
9707SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9708 KnownBits &Known,
9709 const APInt &DemandedElts,
9710 const SelectionDAG &DAG,
9711 unsigned Depth) const {
9712 Known.resetAll();
9713
9714 // Intrinsic CC result is returned in the two low bits.
9715 unsigned Tmp0, Tmp1; // not used
9716 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9717 Known.Zero.setBitsFrom(2);
9718 return;
9719 }
9720 [[maybe_unused]] EVT VT = Op.getValueType();
9721 if (Op.getResNo() != 0)
9722 return;
9723 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9724 "KnownBits does not match VT in bitwidth");
9725 assert ((!VT.isVector() ||
9726 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9727 "DemandedElts does not match VT number of elements");
9728 unsigned BitWidth = Known.getBitWidth();
9729 unsigned Opcode = Op.getOpcode();
9730 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9731 bool IsLogical = false;
9732 unsigned Id = Op.getConstantOperandVal(i: 0);
9733 switch (Id) {
9734 case Intrinsic::s390_vpksh: // PACKS
9735 case Intrinsic::s390_vpksf:
9736 case Intrinsic::s390_vpksg:
9737 case Intrinsic::s390_vpkshs: // PACKS_CC
9738 case Intrinsic::s390_vpksfs:
9739 case Intrinsic::s390_vpksgs:
9740 case Intrinsic::s390_vpklsh: // PACKLS
9741 case Intrinsic::s390_vpklsf:
9742 case Intrinsic::s390_vpklsg:
9743 case Intrinsic::s390_vpklshs: // PACKLS_CC
9744 case Intrinsic::s390_vpklsfs:
9745 case Intrinsic::s390_vpklsgs:
9746 case Intrinsic::s390_vpdi:
9747 case Intrinsic::s390_vsldb:
9748 case Intrinsic::s390_vperm:
9749 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9750 break;
9751 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9752 case Intrinsic::s390_vuplhh:
9753 case Intrinsic::s390_vuplhf:
9754 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9755 case Intrinsic::s390_vupllh:
9756 case Intrinsic::s390_vupllf:
9757 IsLogical = true;
9758 [[fallthrough]];
9759 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9760 case Intrinsic::s390_vuphh:
9761 case Intrinsic::s390_vuphf:
9762 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9763 case Intrinsic::s390_vuplhw:
9764 case Intrinsic::s390_vuplf: {
9765 SDValue SrcOp = Op.getOperand(i: 1);
9766 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9767 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9768 if (IsLogical) {
9769 Known = Known.zext(BitWidth);
9770 } else
9771 Known = Known.sext(BitWidth);
9772 break;
9773 }
9774 default:
9775 break;
9776 }
9777 } else {
9778 switch (Opcode) {
9779 case SystemZISD::JOIN_DWORDS:
9780 case SystemZISD::SELECT_CCMASK:
9781 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9782 break;
9783 case SystemZISD::REPLICATE: {
9784 SDValue SrcOp = Op.getOperand(i: 0);
9785 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9786 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9787 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9788 break;
9789 }
9790 default:
9791 break;
9792 }
9793 }
9794
9795 // Known has the width of the source operand(s). Adjust if needed to match
9796 // the passed bitwidth.
9797 if (Known.getBitWidth() != BitWidth)
9798 Known = Known.anyextOrTrunc(BitWidth);
9799}
9800
9801static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9802 const SelectionDAG &DAG, unsigned Depth,
9803 unsigned OpNo) {
9804 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9805 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9806 if (LHS == 1) return 1; // Early out.
9807 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9808 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9809 if (RHS == 1) return 1; // Early out.
9810 unsigned Common = std::min(a: LHS, b: RHS);
9811 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9812 EVT VT = Op.getValueType();
9813 unsigned VTBits = VT.getScalarSizeInBits();
9814 if (SrcBitWidth > VTBits) { // PACK
9815 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9816 if (Common > SrcExtraBits)
9817 return (Common - SrcExtraBits);
9818 return 1;
9819 }
9820 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9821 return Common;
9822}
9823
9824unsigned
9825SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9826 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9827 unsigned Depth) const {
9828 if (Op.getResNo() != 0)
9829 return 1;
9830 unsigned Opcode = Op.getOpcode();
9831 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9832 unsigned Id = Op.getConstantOperandVal(i: 0);
9833 switch (Id) {
9834 case Intrinsic::s390_vpksh: // PACKS
9835 case Intrinsic::s390_vpksf:
9836 case Intrinsic::s390_vpksg:
9837 case Intrinsic::s390_vpkshs: // PACKS_CC
9838 case Intrinsic::s390_vpksfs:
9839 case Intrinsic::s390_vpksgs:
9840 case Intrinsic::s390_vpklsh: // PACKLS
9841 case Intrinsic::s390_vpklsf:
9842 case Intrinsic::s390_vpklsg:
9843 case Intrinsic::s390_vpklshs: // PACKLS_CC
9844 case Intrinsic::s390_vpklsfs:
9845 case Intrinsic::s390_vpklsgs:
9846 case Intrinsic::s390_vpdi:
9847 case Intrinsic::s390_vsldb:
9848 case Intrinsic::s390_vperm:
9849 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9850 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9851 case Intrinsic::s390_vuphh:
9852 case Intrinsic::s390_vuphf:
9853 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9854 case Intrinsic::s390_vuplhw:
9855 case Intrinsic::s390_vuplf: {
9856 SDValue PackedOp = Op.getOperand(i: 1);
9857 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9858 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9859 EVT VT = Op.getValueType();
9860 unsigned VTBits = VT.getScalarSizeInBits();
9861 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9862 return Tmp;
9863 }
9864 default:
9865 break;
9866 }
9867 } else {
9868 switch (Opcode) {
9869 case SystemZISD::SELECT_CCMASK:
9870 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9871 default:
9872 break;
9873 }
9874 }
9875
9876 return 1;
9877}
9878
9879bool SystemZTargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
9880 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9881 UndefPoisonKind Kind, unsigned Depth) const {
9882 switch (Op->getOpcode()) {
9883 case SystemZISD::PCREL_WRAPPER:
9884 case SystemZISD::PCREL_OFFSET:
9885 return true;
9886 }
9887 return false;
9888}
9889
9890unsigned
9891SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9892 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9893 unsigned StackAlign = TFI->getStackAlignment();
9894 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9895 "Unexpected stack alignment");
9896 // The default stack probe size is 4096 if the function has no
9897 // stack-probe-size attribute.
9898 unsigned StackProbeSize =
9899 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9900 // Round down to the stack alignment.
9901 StackProbeSize &= ~(StackAlign - 1);
9902 return StackProbeSize ? StackProbeSize : StackAlign;
9903}
9904
9905//===----------------------------------------------------------------------===//
9906// Custom insertion
9907//===----------------------------------------------------------------------===//
9908
9909// Force base value Base into a register before MI. Return the register.
9910static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9911 const SystemZInstrInfo *TII) {
9912 MachineBasicBlock *MBB = MI.getParent();
9913 MachineFunction &MF = *MBB->getParent();
9914 MachineRegisterInfo &MRI = MF.getRegInfo();
9915
9916 if (Base.isReg()) {
9917 // Copy Base into a new virtual register to help register coalescing in
9918 // cases with multiple uses.
9919 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9920 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9921 .add(MO: Base);
9922 return Reg;
9923 }
9924
9925 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9926 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9927 .add(MO: Base)
9928 .addImm(Val: 0)
9929 .addReg(RegNo: 0);
9930 return Reg;
9931}
9932
9933// The CC operand of MI might be missing a kill marker because there
9934// were multiple uses of CC, and ISel didn't know which to mark.
9935// Figure out whether MI should have had a kill marker.
9936static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9937 // Scan forward through BB for a use/def of CC.
9938 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9939 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9940 const MachineInstr &MI = *miI;
9941 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9942 return false;
9943 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9944 break; // Should have kill-flag - update below.
9945 }
9946
9947 // If we hit the end of the block, check whether CC is live into a
9948 // successor.
9949 if (miI == MBB->end()) {
9950 for (const MachineBasicBlock *Succ : MBB->successors())
9951 if (Succ->isLiveIn(Reg: SystemZ::CC))
9952 return false;
9953 }
9954
9955 return true;
9956}
9957
9958// Return true if it is OK for this Select pseudo-opcode to be cascaded
9959// together with other Select pseudo-opcodes into a single basic-block with
9960// a conditional jump around it.
9961static bool isSelectPseudo(MachineInstr &MI) {
9962 switch (MI.getOpcode()) {
9963 case SystemZ::Select32:
9964 case SystemZ::Select64:
9965 case SystemZ::Select128:
9966 case SystemZ::SelectF32:
9967 case SystemZ::SelectF64:
9968 case SystemZ::SelectF128:
9969 case SystemZ::SelectVR32:
9970 case SystemZ::SelectVR64:
9971 case SystemZ::SelectVR128:
9972 return true;
9973
9974 default:
9975 return false;
9976 }
9977}
9978
9979// Helper function, which inserts PHI functions into SinkMBB:
9980// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9981// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9982static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9983 MachineBasicBlock *TrueMBB,
9984 MachineBasicBlock *FalseMBB,
9985 MachineBasicBlock *SinkMBB) {
9986 MachineFunction *MF = TrueMBB->getParent();
9987 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9988
9989 MachineInstr *FirstMI = Selects.front();
9990 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9991 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9992
9993 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9994
9995 // As we are creating the PHIs, we have to be careful if there is more than
9996 // one. Later Selects may reference the results of earlier Selects, but later
9997 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9998 // That also means that PHI construction must work forward from earlier to
9999 // later, and that the code must maintain a mapping from earlier PHI's
10000 // destination registers, and the registers that went into the PHI.
10001 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
10002
10003 for (auto *MI : Selects) {
10004 Register DestReg = MI->getOperand(i: 0).getReg();
10005 Register TrueReg = MI->getOperand(i: 1).getReg();
10006 Register FalseReg = MI->getOperand(i: 2).getReg();
10007
10008 // If this Select we are generating is the opposite condition from
10009 // the jump we generated, then we have to swap the operands for the
10010 // PHI that is going to be generated.
10011 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
10012 std::swap(a&: TrueReg, b&: FalseReg);
10013
10014 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
10015 TrueReg = It->second.first;
10016
10017 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
10018 FalseReg = It->second.second;
10019
10020 DebugLoc DL = MI->getDebugLoc();
10021 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
10022 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
10023 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
10024
10025 // Add this PHI to the rewrite table.
10026 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
10027 }
10028
10029 MF->getProperties().resetNoPHIs();
10030}
10031
10032MachineBasicBlock *
10033SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
10034 MachineBasicBlock *BB) const {
10035 MachineFunction &MF = *BB->getParent();
10036 MachineFrameInfo &MFI = MF.getFrameInfo();
10037 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
10038 assert(TFL->hasReservedCallFrame(MF) &&
10039 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
10040 (void)TFL;
10041 // Get the MaxCallFrameSize value and erase MI since it serves no further
10042 // purpose as the call frame is statically reserved in the prolog. Set
10043 // AdjustsStack as MI is *not* mapped as a frame instruction.
10044 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
10045 if (NumBytes > MFI.getMaxCallFrameSize())
10046 MFI.setMaxCallFrameSize(NumBytes);
10047 MFI.setAdjustsStack(true);
10048
10049 MI.eraseFromParent();
10050 return BB;
10051}
10052
10053// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
10054MachineBasicBlock *
10055SystemZTargetLowering::emitSelect(MachineInstr &MI,
10056 MachineBasicBlock *MBB) const {
10057 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
10058 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10059
10060 unsigned CCValid = MI.getOperand(i: 3).getImm();
10061 unsigned CCMask = MI.getOperand(i: 4).getImm();
10062
10063 // If we have a sequence of Select* pseudo instructions using the
10064 // same condition code value, we want to expand all of them into
10065 // a single pair of basic blocks using the same condition.
10066 SmallVector<MachineInstr*, 8> Selects;
10067 SmallVector<MachineInstr*, 8> DbgValues;
10068 Selects.push_back(Elt: &MI);
10069 unsigned Count = 0;
10070 for (MachineInstr &NextMI : llvm::make_range(
10071 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
10072 if (isSelectPseudo(MI&: NextMI)) {
10073 assert(NextMI.getOperand(3).getImm() == CCValid &&
10074 "Bad CCValid operands since CC was not redefined.");
10075 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
10076 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
10077 Selects.push_back(Elt: &NextMI);
10078 continue;
10079 }
10080 break;
10081 }
10082 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
10083 NextMI.usesCustomInsertionHook())
10084 break;
10085 bool User = false;
10086 for (auto *SelMI : Selects)
10087 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
10088 User = true;
10089 break;
10090 }
10091 if (NextMI.isDebugInstr()) {
10092 if (User) {
10093 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
10094 DbgValues.push_back(Elt: &NextMI);
10095 }
10096 } else if (User || ++Count > 20)
10097 break;
10098 }
10099
10100 MachineInstr *LastMI = Selects.back();
10101 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
10102 checkCCKill(MI&: *LastMI, MBB));
10103 MachineBasicBlock *StartMBB = MBB;
10104 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
10105 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10106
10107 // Unless CC was killed in the last Select instruction, mark it as
10108 // live-in to both FalseMBB and JoinMBB.
10109 if (!CCKilled) {
10110 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10111 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10112 }
10113
10114 // StartMBB:
10115 // BRC CCMask, JoinMBB
10116 // # fallthrough to FalseMBB
10117 MBB = StartMBB;
10118 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10119 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10120 MBB->addSuccessor(Succ: JoinMBB);
10121 MBB->addSuccessor(Succ: FalseMBB);
10122
10123 // FalseMBB:
10124 // # fallthrough to JoinMBB
10125 MBB = FalseMBB;
10126 MBB->addSuccessor(Succ: JoinMBB);
10127
10128 // JoinMBB:
10129 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
10130 // ...
10131 MBB = JoinMBB;
10132 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
10133 for (auto *SelMI : Selects)
10134 SelMI->eraseFromParent();
10135
10136 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
10137 for (auto *DbgMI : DbgValues)
10138 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
10139
10140 return JoinMBB;
10141}
10142
10143// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
10144// StoreOpcode is the store to use and Invert says whether the store should
10145// happen when the condition is false rather than true. If a STORE ON
10146// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
10147MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
10148 MachineBasicBlock *MBB,
10149 unsigned StoreOpcode,
10150 unsigned STOCOpcode,
10151 bool Invert) const {
10152 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10153
10154 Register SrcReg = MI.getOperand(i: 0).getReg();
10155 MachineOperand Base = MI.getOperand(i: 1);
10156 int64_t Disp = MI.getOperand(i: 2).getImm();
10157 Register IndexReg = MI.getOperand(i: 3).getReg();
10158 unsigned CCValid = MI.getOperand(i: 4).getImm();
10159 unsigned CCMask = MI.getOperand(i: 5).getImm();
10160 DebugLoc DL = MI.getDebugLoc();
10161
10162 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
10163
10164 // ISel pattern matching also adds a load memory operand of the same
10165 // address, so take special care to find the storing memory operand.
10166 MachineMemOperand *MMO = nullptr;
10167 for (auto *I : MI.memoperands())
10168 if (I->isStore()) {
10169 MMO = I;
10170 break;
10171 }
10172
10173 // Use STOCOpcode if possible. We could use different store patterns in
10174 // order to avoid matching the index register, but the performance trade-offs
10175 // might be more complicated in that case.
10176 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10177 if (Invert)
10178 CCMask ^= CCValid;
10179
10180 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
10181 .addReg(RegNo: SrcReg)
10182 .add(MO: Base)
10183 .addImm(Val: Disp)
10184 .addImm(Val: CCValid)
10185 .addImm(Val: CCMask)
10186 .addMemOperand(MMO);
10187
10188 MI.eraseFromParent();
10189 return MBB;
10190 }
10191
10192 // Get the condition needed to branch around the store.
10193 if (!Invert)
10194 CCMask ^= CCValid;
10195
10196 MachineBasicBlock *StartMBB = MBB;
10197 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
10198 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10199
10200 // Unless CC was killed in the CondStore instruction, mark it as
10201 // live-in to both FalseMBB and JoinMBB.
10202 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
10203 !checkCCKill(MI, MBB: JoinMBB)) {
10204 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10205 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10206 }
10207
10208 // StartMBB:
10209 // BRC CCMask, JoinMBB
10210 // # fallthrough to FalseMBB
10211 MBB = StartMBB;
10212 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10213 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10214 MBB->addSuccessor(Succ: JoinMBB);
10215 MBB->addSuccessor(Succ: FalseMBB);
10216
10217 // FalseMBB:
10218 // store %SrcReg, %Disp(%Index,%Base)
10219 // # fallthrough to JoinMBB
10220 MBB = FalseMBB;
10221 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
10222 .addReg(RegNo: SrcReg)
10223 .add(MO: Base)
10224 .addImm(Val: Disp)
10225 .addReg(RegNo: IndexReg)
10226 .addMemOperand(MMO);
10227 MBB->addSuccessor(Succ: JoinMBB);
10228
10229 MI.eraseFromParent();
10230 return JoinMBB;
10231}
10232
10233// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10234MachineBasicBlock *
10235SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10236 MachineBasicBlock *MBB,
10237 bool Unsigned) const {
10238 MachineFunction &MF = *MBB->getParent();
10239 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10240 MachineRegisterInfo &MRI = MF.getRegInfo();
10241
10242 // Synthetic instruction to compare 128-bit values.
10243 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10244 Register Op0 = MI.getOperand(i: 0).getReg();
10245 Register Op1 = MI.getOperand(i: 1).getReg();
10246
10247 MachineBasicBlock *StartMBB = MBB;
10248 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10249 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10250
10251 // StartMBB:
10252 //
10253 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10254 // Swap the inputs to get:
10255 // CC 1 if high(Op0) > high(Op1)
10256 // CC 2 if high(Op0) < high(Op1)
10257 // CC 0 if high(Op0) == high(Op1)
10258 //
10259 // If CC != 0, we'd done, so jump over the next instruction.
10260 //
10261 // VEC[L]G Op1, Op0
10262 // JNE JoinMBB
10263 // # fallthrough to HiEqMBB
10264 MBB = StartMBB;
10265 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10266 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
10267 .addReg(RegNo: Op1).addReg(RegNo: Op0);
10268 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10269 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
10270 MBB->addSuccessor(Succ: JoinMBB);
10271 MBB->addSuccessor(Succ: HiEqMBB);
10272
10273 // HiEqMBB:
10274 //
10275 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10276 // Since we already know the high parts are equal, the CC
10277 // result will only depend on the low parts:
10278 // CC 1 if low(Op0) > low(Op1)
10279 // CC 3 if low(Op0) <= low(Op1)
10280 //
10281 // VCHLGS Tmp, Op0, Op1
10282 // # fallthrough to JoinMBB
10283 MBB = HiEqMBB;
10284 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
10285 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
10286 .addReg(RegNo: Op0).addReg(RegNo: Op1);
10287 MBB->addSuccessor(Succ: JoinMBB);
10288
10289 // Mark CC as live-in to JoinMBB.
10290 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10291
10292 MI.eraseFromParent();
10293 return JoinMBB;
10294}
10295
10296// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10297// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10298// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10299// whether the field should be inverted after performing BinOpcode (e.g. for
10300// NAND).
10301MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10302 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10303 bool Invert) const {
10304 MachineFunction &MF = *MBB->getParent();
10305 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10306 MachineRegisterInfo &MRI = MF.getRegInfo();
10307
10308 // Extract the operands. Base can be a register or a frame index.
10309 // Src2 can be a register or immediate.
10310 Register Dest = MI.getOperand(i: 0).getReg();
10311 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10312 int64_t Disp = MI.getOperand(i: 2).getImm();
10313 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
10314 Register BitShift = MI.getOperand(i: 4).getReg();
10315 Register NegBitShift = MI.getOperand(i: 5).getReg();
10316 unsigned BitSize = MI.getOperand(i: 6).getImm();
10317 DebugLoc DL = MI.getDebugLoc();
10318
10319 // Get the right opcodes for the displacement.
10320 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10321 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10322 assert(LOpcode && CSOpcode && "Displacement out of range");
10323
10324 // Create virtual registers for temporary results.
10325 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10326 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10327 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10328 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10329 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10330
10331 // Insert a basic block for the main loop.
10332 MachineBasicBlock *StartMBB = MBB;
10333 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10334 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10335
10336 // StartMBB:
10337 // ...
10338 // %OrigVal = L Disp(%Base)
10339 // # fall through to LoopMBB
10340 MBB = StartMBB;
10341 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10342 MBB->addSuccessor(Succ: LoopMBB);
10343
10344 // LoopMBB:
10345 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10346 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10347 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10348 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10349 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10350 // JNE LoopMBB
10351 // # fall through to DoneMBB
10352 MBB = LoopMBB;
10353 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10354 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10355 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
10356 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10357 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10358 if (Invert) {
10359 // Perform the operation normally and then invert every bit of the field.
10360 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10361 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp)
10362 .addReg(RegNo: RotatedOldVal)
10363 .add(MO: Src2)
10364 .setOperandDead(3);
10365 // XILF with the upper BitSize bits set.
10366 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
10367 .addReg(RegNo: Tmp)
10368 .addImm(Val: -1U << (32 - BitSize))
10369 .setOperandDead(3);
10370 } else if (BinOpcode)
10371 // A simply binary operation.
10372 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
10373 .addReg(RegNo: RotatedOldVal)
10374 .add(MO: Src2)
10375 .setOperandDead(3);
10376 else
10377 // Use RISBG to rotate Src2 into position and use it to replace the
10378 // field in RotatedOldVal.
10379 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
10380 .addReg(RegNo: RotatedOldVal)
10381 .addReg(RegNo: Src2.getReg())
10382 .addImm(Val: 32)
10383 .addImm(Val: 31 + BitSize)
10384 .addImm(Val: 32 - BitSize)
10385 .setOperandDead(6);
10386 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10387 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10388 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10389 .addReg(RegNo: OldVal)
10390 .addReg(RegNo: NewVal)
10391 .add(MO: Base)
10392 .addImm(Val: Disp);
10393 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10394 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10395 MBB->addSuccessor(Succ: LoopMBB);
10396 MBB->addSuccessor(Succ: DoneMBB);
10397
10398 MI.eraseFromParent();
10399 return DoneMBB;
10400}
10401
10402// Implement EmitInstrWithCustomInserter for subword pseudo
10403// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10404// instruction that should be used to compare the current field with the
10405// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10406// for when the current field should be kept.
10407MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10408 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10409 unsigned KeepOldMask) const {
10410 MachineFunction &MF = *MBB->getParent();
10411 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10412 MachineRegisterInfo &MRI = MF.getRegInfo();
10413
10414 // Extract the operands. Base can be a register or a frame index.
10415 Register Dest = MI.getOperand(i: 0).getReg();
10416 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10417 int64_t Disp = MI.getOperand(i: 2).getImm();
10418 Register Src2 = MI.getOperand(i: 3).getReg();
10419 Register BitShift = MI.getOperand(i: 4).getReg();
10420 Register NegBitShift = MI.getOperand(i: 5).getReg();
10421 unsigned BitSize = MI.getOperand(i: 6).getImm();
10422 DebugLoc DL = MI.getDebugLoc();
10423
10424 // Get the right opcodes for the displacement.
10425 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10426 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10427 assert(LOpcode && CSOpcode && "Displacement out of range");
10428
10429 // Create virtual registers for temporary results.
10430 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10431 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10432 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10433 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10434 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10435 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10436
10437 // Insert 3 basic blocks for the loop.
10438 MachineBasicBlock *StartMBB = MBB;
10439 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10440 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10441 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10442 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10443
10444 // StartMBB:
10445 // ...
10446 // %OrigVal = L Disp(%Base)
10447 // # fall through to LoopMBB
10448 MBB = StartMBB;
10449 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10450 MBB->addSuccessor(Succ: LoopMBB);
10451
10452 // LoopMBB:
10453 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10454 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10455 // CompareOpcode %RotatedOldVal, %Src2
10456 // BRC KeepOldMask, UpdateMBB
10457 MBB = LoopMBB;
10458 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10459 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10460 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10461 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10462 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10463 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10464 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10465 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10466 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10467 MBB->addSuccessor(Succ: UpdateMBB);
10468 MBB->addSuccessor(Succ: UseAltMBB);
10469
10470 // UseAltMBB:
10471 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10472 // # fall through to UpdateMBB
10473 MBB = UseAltMBB;
10474 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10475 .addReg(RegNo: RotatedOldVal)
10476 .addReg(RegNo: Src2)
10477 .addImm(Val: 32)
10478 .addImm(Val: 31 + BitSize)
10479 .addImm(Val: 0)
10480 .setOperandDead(6);
10481 MBB->addSuccessor(Succ: UpdateMBB);
10482
10483 // UpdateMBB:
10484 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10485 // [ %RotatedAltVal, UseAltMBB ]
10486 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10487 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10488 // JNE LoopMBB
10489 // # fall through to DoneMBB
10490 MBB = UpdateMBB;
10491 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10492 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10493 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10494 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10495 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10496 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10497 .addReg(RegNo: OldVal)
10498 .addReg(RegNo: NewVal)
10499 .add(MO: Base)
10500 .addImm(Val: Disp);
10501 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10502 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10503 MBB->addSuccessor(Succ: LoopMBB);
10504 MBB->addSuccessor(Succ: DoneMBB);
10505
10506 MI.eraseFromParent();
10507 return DoneMBB;
10508}
10509
10510// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10511// instruction MI.
10512MachineBasicBlock *
10513SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10514 MachineBasicBlock *MBB) const {
10515 MachineFunction &MF = *MBB->getParent();
10516 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10517 MachineRegisterInfo &MRI = MF.getRegInfo();
10518
10519 // Extract the operands. Base can be a register or a frame index.
10520 Register Dest = MI.getOperand(i: 0).getReg();
10521 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10522 int64_t Disp = MI.getOperand(i: 2).getImm();
10523 Register CmpVal = MI.getOperand(i: 3).getReg();
10524 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10525 Register BitShift = MI.getOperand(i: 5).getReg();
10526 Register NegBitShift = MI.getOperand(i: 6).getReg();
10527 int64_t BitSize = MI.getOperand(i: 7).getImm();
10528 DebugLoc DL = MI.getDebugLoc();
10529
10530 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10531
10532 // Get the right opcodes for the displacement and zero-extension.
10533 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10534 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10535 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10536 assert(LOpcode && CSOpcode && "Displacement out of range");
10537
10538 // Create virtual registers for temporary results.
10539 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10540 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10541 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10542 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10543 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10544 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10545 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10546
10547 // Insert 2 basic blocks for the loop.
10548 MachineBasicBlock *StartMBB = MBB;
10549 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10550 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10551 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10552
10553 // StartMBB:
10554 // ...
10555 // %OrigOldVal = L Disp(%Base)
10556 // # fall through to LoopMBB
10557 MBB = StartMBB;
10558 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10559 .add(MO: Base)
10560 .addImm(Val: Disp)
10561 .addReg(RegNo: 0);
10562 MBB->addSuccessor(Succ: LoopMBB);
10563
10564 // LoopMBB:
10565 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10566 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10567 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10568 // ^^ The low BitSize bits contain the field
10569 // of interest.
10570 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10571 // ^^ Replace the upper 32-BitSize bits of the
10572 // swap value with those that we loaded and rotated.
10573 // %Dest = LL[CH] %OldValRot
10574 // CR %Dest, %CmpVal
10575 // JNE DoneMBB
10576 // # Fall through to SetMBB
10577 MBB = LoopMBB;
10578 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10579 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10580 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10581 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10582 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10583 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10584 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10585 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10586 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10587 .addReg(RegNo: SwapVal)
10588 .addReg(RegNo: OldValRot)
10589 .addImm(Val: 32)
10590 .addImm(Val: 63 - BitSize)
10591 .addImm(Val: 0)
10592 .setOperandDead(6);
10593 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10594 .addReg(RegNo: OldValRot);
10595 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10596 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10597 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10598 .addImm(Val: SystemZ::CCMASK_ICMP)
10599 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10600 MBB->addSuccessor(Succ: DoneMBB);
10601 MBB->addSuccessor(Succ: SetMBB);
10602
10603 // SetMBB:
10604 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10605 // ^^ Rotate the new field to its proper position.
10606 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10607 // JNE LoopMBB
10608 // # fall through to ExitMBB
10609 MBB = SetMBB;
10610 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10611 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10612 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10613 .addReg(RegNo: OldVal)
10614 .addReg(RegNo: StoreVal)
10615 .add(MO: Base)
10616 .addImm(Val: Disp);
10617 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10618 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10619 MBB->addSuccessor(Succ: LoopMBB);
10620 MBB->addSuccessor(Succ: DoneMBB);
10621
10622 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10623 // to the block after the loop. At this point, CC may have been defined
10624 // either by the CR in LoopMBB or by the CS in SetMBB.
10625 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10626 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10627
10628 MI.eraseFromParent();
10629 return DoneMBB;
10630}
10631
10632MachineBasicBlock *
10633SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10634 MachineBasicBlock *MBB,
10635 unsigned Opcode, bool IsMemset) const {
10636 MachineFunction &MF = *MBB->getParent();
10637 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10638 MachineRegisterInfo &MRI = MF.getRegInfo();
10639 DebugLoc DL = MI.getDebugLoc();
10640
10641 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10642 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10643 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10644 uint64_t SrcDisp;
10645
10646 // Fold the displacement Disp if it is out of range.
10647 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10648 if (!isUInt<12>(x: Disp)) {
10649 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10650 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10651 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10652 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10653 Base = MachineOperand::CreateReg(Reg, isDef: false);
10654 Disp = 0;
10655 }
10656 };
10657
10658 if (!IsMemset) {
10659 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10660 SrcDisp = MI.getOperand(i: 3).getImm();
10661 } else {
10662 SrcBase = DestBase;
10663 SrcDisp = DestDisp++;
10664 foldDisplIfNeeded(DestBase, DestDisp);
10665 }
10666
10667 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10668 bool IsImmForm = LengthMO.isImm();
10669 bool IsRegForm = !IsImmForm;
10670
10671 // Build and insert one Opcode of Length, with special treatment for memset.
10672 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10673 MachineBasicBlock::iterator InsPos,
10674 MachineOperand DBase, uint64_t DDisp,
10675 MachineOperand SBase, uint64_t SDisp,
10676 unsigned Length) -> void {
10677 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10678 if (IsMemset) {
10679 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10680 if (ByteMO.isImm())
10681 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10682 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10683 else
10684 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10685 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10686 if (--Length == 0)
10687 return;
10688 }
10689 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10690 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10691 .add(MO: SBase).addImm(Val: SDisp)
10692 .setMemRefs(MI.memoperands());
10693 };
10694
10695 bool NeedsLoop = false;
10696 uint64_t ImmLength = 0;
10697 Register LenAdjReg = SystemZ::NoRegister;
10698 if (IsImmForm) {
10699 ImmLength = LengthMO.getImm();
10700 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10701 if (ImmLength == 0) {
10702 MI.eraseFromParent();
10703 return MBB;
10704 }
10705 if (Opcode == SystemZ::CLC) {
10706 if (ImmLength > 3 * 256)
10707 // A two-CLC sequence is a clear win over a loop, not least because
10708 // it needs only one branch. A three-CLC sequence needs the same
10709 // number of branches as a loop (i.e. 2), but is shorter. That
10710 // brings us to lengths greater than 768 bytes. It seems relatively
10711 // likely that a difference will be found within the first 768 bytes,
10712 // so we just optimize for the smallest number of branch
10713 // instructions, in order to avoid polluting the prediction buffer
10714 // too much.
10715 NeedsLoop = true;
10716 } else if (ImmLength > 6 * 256)
10717 // The heuristic we use is to prefer loops for anything that would
10718 // require 7 or more MVCs. With these kinds of sizes there isn't much
10719 // to choose between straight-line code and looping code, since the
10720 // time will be dominated by the MVCs themselves.
10721 NeedsLoop = true;
10722 } else {
10723 NeedsLoop = true;
10724 LenAdjReg = LengthMO.getReg();
10725 }
10726
10727 // When generating more than one CLC, all but the last will need to
10728 // branch to the end when a difference is found.
10729 MachineBasicBlock *EndMBB =
10730 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10731 ? SystemZ::splitBlockAfter(MI, MBB)
10732 : nullptr);
10733
10734 if (NeedsLoop) {
10735 Register StartCountReg =
10736 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10737 if (IsImmForm) {
10738 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10739 ImmLength &= 255;
10740 } else {
10741 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10742 .addReg(RegNo: LenAdjReg)
10743 .addReg(RegNo: 0)
10744 .addImm(Val: 8);
10745 }
10746
10747 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10748 auto loadZeroAddress = [&]() -> MachineOperand {
10749 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10750 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10751 return MachineOperand::CreateReg(Reg, isDef: false);
10752 };
10753 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10754 DestBase = loadZeroAddress();
10755 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10756 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10757
10758 MachineBasicBlock *StartMBB = nullptr;
10759 MachineBasicBlock *LoopMBB = nullptr;
10760 MachineBasicBlock *NextMBB = nullptr;
10761 MachineBasicBlock *DoneMBB = nullptr;
10762 MachineBasicBlock *AllDoneMBB = nullptr;
10763
10764 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10765 Register StartDestReg =
10766 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10767
10768 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10769 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10770 Register ThisDestReg =
10771 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10772 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10773 Register NextDestReg =
10774 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10775 RC = &SystemZ::GR64BitRegClass;
10776 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10777 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10778
10779 if (IsRegForm) {
10780 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10781 StartMBB = SystemZ::emitBlockAfter(MBB);
10782 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10783 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10784 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10785
10786 // MBB:
10787 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10788 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10789 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10790 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10791 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10792 .addMBB(MBB: AllDoneMBB);
10793 MBB->addSuccessor(Succ: AllDoneMBB);
10794 if (!IsMemset)
10795 MBB->addSuccessor(Succ: StartMBB);
10796 else {
10797 // MemsetOneCheckMBB:
10798 // # Jump to MemsetOneMBB for a memset of length 1, or
10799 // # fall thru to StartMBB.
10800 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10801 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10802 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10803 MBB = MemsetOneCheckMBB;
10804 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10805 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10806 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10807 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10808 .addMBB(MBB: MemsetOneMBB);
10809 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10810 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10811
10812 // MemsetOneMBB:
10813 // # Jump back to AllDoneMBB after a single MVI or STC.
10814 MBB = MemsetOneMBB;
10815 insertMemMemOp(MBB, MBB->end(),
10816 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10817 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10818 1);
10819 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10820 MBB->addSuccessor(Succ: AllDoneMBB);
10821 }
10822
10823 // StartMBB:
10824 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10825 MBB = StartMBB;
10826 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10827 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10828 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10829 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10830 .addMBB(MBB: DoneMBB);
10831 MBB->addSuccessor(Succ: DoneMBB);
10832 MBB->addSuccessor(Succ: LoopMBB);
10833 }
10834 else {
10835 StartMBB = MBB;
10836 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10837 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10838 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10839
10840 // StartMBB:
10841 // # fall through to LoopMBB
10842 MBB->addSuccessor(Succ: LoopMBB);
10843
10844 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10845 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10846 if (EndMBB && !ImmLength)
10847 // If the loop handled the whole CLC range, DoneMBB will be empty with
10848 // CC live-through into EndMBB, so add it as live-in.
10849 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10850 }
10851
10852 // LoopMBB:
10853 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10854 // [ %NextDestReg, NextMBB ]
10855 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10856 // [ %NextSrcReg, NextMBB ]
10857 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10858 // [ %NextCountReg, NextMBB ]
10859 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10860 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10861 // ( JLH EndMBB )
10862 //
10863 // The prefetch is used only for MVC. The JLH is used only for CLC.
10864 MBB = LoopMBB;
10865 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10866 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10867 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10868 if (!HaveSingleBase)
10869 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10870 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10871 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10872 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10873 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10874 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10875 if (Opcode == SystemZ::MVC)
10876 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10877 .addImm(Val: SystemZ::PFD_WRITE)
10878 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10879 insertMemMemOp(MBB, MBB->end(),
10880 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10881 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10882 if (EndMBB) {
10883 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10884 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10885 .addMBB(MBB: EndMBB);
10886 MBB->addSuccessor(Succ: EndMBB);
10887 MBB->addSuccessor(Succ: NextMBB);
10888 }
10889
10890 // NextMBB:
10891 // %NextDestReg = LA 256(%ThisDestReg)
10892 // %NextSrcReg = LA 256(%ThisSrcReg)
10893 // %NextCountReg = AGHI %ThisCountReg, -1
10894 // CGHI %NextCountReg, 0
10895 // JLH LoopMBB
10896 // # fall through to DoneMBB
10897 //
10898 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10899 MBB = NextMBB;
10900 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10901 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10902 if (!HaveSingleBase)
10903 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10904 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10905 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10906 .addReg(RegNo: ThisCountReg)
10907 .addImm(Val: -1)
10908 .setOperandDead(3);
10909 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10910 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10911 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10912 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10913 .addMBB(MBB: LoopMBB);
10914 MBB->addSuccessor(Succ: LoopMBB);
10915 MBB->addSuccessor(Succ: DoneMBB);
10916
10917 MBB = DoneMBB;
10918 if (IsRegForm) {
10919 // DoneMBB:
10920 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10921 // # Use EXecute Relative Long for the remainder of the bytes. The target
10922 // instruction of the EXRL will have a length field of 1 since 0 is an
10923 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10924 // 0xff) + 1.
10925 // # Fall through to AllDoneMBB.
10926 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10927 Register RemDestReg = HaveSingleBase ? RemSrcReg
10928 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10929 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10930 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10931 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10932 if (!HaveSingleBase)
10933 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10934 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10935 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10936 if (IsMemset)
10937 insertMemMemOp(MBB, MBB->end(),
10938 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10939 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10940 MachineInstrBuilder EXRL_MIB =
10941 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10942 .addImm(Val: Opcode)
10943 .addReg(RegNo: LenAdjReg)
10944 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10945 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10946 MBB->addSuccessor(Succ: AllDoneMBB);
10947 MBB = AllDoneMBB;
10948 if (Opcode != SystemZ::MVC) {
10949 EXRL_MIB.addReg(RegNo: SystemZ::CC, Flags: RegState::ImplicitDefine);
10950 if (EndMBB)
10951 MBB->addLiveIn(PhysReg: SystemZ::CC);
10952 }
10953 }
10954 MF.getProperties().resetNoPHIs();
10955 }
10956
10957 // Handle any remaining bytes with straight-line code.
10958 while (ImmLength > 0) {
10959 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10960 // The previous iteration might have created out-of-range displacements.
10961 // Apply them using LA/LAY if so.
10962 foldDisplIfNeeded(DestBase, DestDisp);
10963 foldDisplIfNeeded(SrcBase, SrcDisp);
10964 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10965 DestDisp += ThisLength;
10966 SrcDisp += ThisLength;
10967 ImmLength -= ThisLength;
10968 // If there's another CLC to go, branch to the end if a difference
10969 // was found.
10970 if (EndMBB && ImmLength > 0) {
10971 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10972 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10973 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10974 .addMBB(MBB: EndMBB);
10975 MBB->addSuccessor(Succ: EndMBB);
10976 MBB->addSuccessor(Succ: NextMBB);
10977 MBB = NextMBB;
10978 }
10979 }
10980 if (EndMBB) {
10981 MBB->addSuccessor(Succ: EndMBB);
10982 MBB = EndMBB;
10983 MBB->addLiveIn(PhysReg: SystemZ::CC);
10984 }
10985
10986 MI.eraseFromParent();
10987 return MBB;
10988}
10989
10990MachineBasicBlock *
10991SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
10992 MachineBasicBlock *MBB) const {
10993 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10994
10995 DebugLoc DL = MI.getDebugLoc();
10996 MachineOperand DstAddr = earlyUseOperand(Op: MI.getOperand(i: 0));
10997 MachineOperand SrcAddr = earlyUseOperand(Op: MI.getOperand(i: 1));
10998 uint64_t Len = MI.getOperand(i: 2).getImm();
10999 assert(Len > 0 && Len <= 256 && "Memmove of of unsupported constant length.");
11000
11001 // Use MVC or MVCRL after comparing the addresses.
11002 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11003 MachineBasicBlock *MvcMBB = SystemZ::emitBlockAfter(MBB);
11004 MachineBasicBlock *MvcrlMBB = SystemZ::emitBlockAfter(MBB: MvcMBB);
11005 MBB->addSuccessor(Succ: MvcMBB);
11006 MBB->addSuccessor(Succ: MvcrlMBB);
11007 MvcMBB->addSuccessor(Succ: DoneMBB);
11008 MvcrlMBB->addSuccessor(Succ: DoneMBB);
11009
11010 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGR)).add(MO: SrcAddr).add(MO: DstAddr);
11011 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11012 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
11013 .addMBB(MBB: MvcrlMBB);
11014
11015 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVC))
11016 .add(MO: DstAddr).addImm(Val: 0)
11017 .addImm(Val: Len)
11018 .add(MO: SrcAddr).addImm(Val: 0)
11019 .setMemRefs(MI.memoperands());
11020 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: DoneMBB);
11021
11022 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: SystemZ::R0L).addImm(Val: Len - 1);
11023 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVCRL))
11024 .add(MO: DstAddr).addImm(Val: 0)
11025 .add(MO: SrcAddr).addImm(Val: 0)
11026 .setMemRefs(MI.memoperands());
11027
11028 MI.eraseFromParent();
11029 return DoneMBB;
11030}
11031
11032// Decompose string pseudo-instruction MI into a loop that continually performs
11033// Opcode until CC != 3.
11034MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
11035 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11036 MachineFunction &MF = *MBB->getParent();
11037 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11038 MachineRegisterInfo &MRI = MF.getRegInfo();
11039 DebugLoc DL = MI.getDebugLoc();
11040
11041 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
11042 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
11043 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
11044 uint64_t CharReg = MI.getOperand(i: 3).getReg();
11045
11046 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
11047 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
11048 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
11049 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
11050
11051 MachineBasicBlock *StartMBB = MBB;
11052 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
11053 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11054
11055 // StartMBB:
11056 // # fall through to LoopMBB
11057 MBB->addSuccessor(Succ: LoopMBB);
11058
11059 // LoopMBB:
11060 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
11061 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
11062 // R0L = %CharReg
11063 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
11064 // JO LoopMBB
11065 // # fall through to DoneMBB
11066 //
11067 // The load of R0L can be hoisted by post-RA LICM.
11068 MBB = LoopMBB;
11069
11070 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
11071 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
11072 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
11073 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
11074 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
11075 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
11076 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
11077 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
11078 .addReg(RegNo: End1Reg, Flags: RegState::Define).addReg(RegNo: End2Reg, Flags: RegState::Define)
11079 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
11080 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11081 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
11082 MBB->addSuccessor(Succ: LoopMBB);
11083 MBB->addSuccessor(Succ: DoneMBB);
11084
11085 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
11086
11087 MI.eraseFromParent();
11088 return DoneMBB;
11089}
11090
11091// Update TBEGIN instruction with final opcode and register clobbers.
11092MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
11093 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
11094 bool NoFloat) const {
11095 MachineFunction &MF = *MBB->getParent();
11096 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11097 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11098
11099 // Update opcode.
11100 MI.setDesc(TII->get(Opcode));
11101
11102 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
11103 // Make sure to add the corresponding GRSM bits if they are missing.
11104 uint64_t Control = MI.getOperand(i: 2).getImm();
11105 static const unsigned GPRControlBit[16] = {
11106 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
11107 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
11108 };
11109 Control |= GPRControlBit[15];
11110 if (TFI->hasFP(MF))
11111 Control |= GPRControlBit[11];
11112 MI.getOperand(i: 2).setImm(Control);
11113
11114 // Add GPR clobbers.
11115 for (int I = 0; I < 16; I++) {
11116 if ((Control & GPRControlBit[I]) == 0) {
11117 unsigned Reg = SystemZMC::GR64Regs[I];
11118 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11119 }
11120 }
11121
11122 // Add FPR/VR clobbers.
11123 if (!NoFloat && (Control & 4) != 0) {
11124 if (Subtarget.hasVector()) {
11125 for (unsigned Reg : SystemZMC::VR128Regs) {
11126 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11127 }
11128 } else {
11129 for (unsigned Reg : SystemZMC::FP64Regs) {
11130 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11131 }
11132 }
11133 }
11134
11135 return MBB;
11136}
11137
11138MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
11139 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11140 MachineFunction &MF = *MBB->getParent();
11141 MachineRegisterInfo *MRI = &MF.getRegInfo();
11142 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11143 DebugLoc DL = MI.getDebugLoc();
11144
11145 Register SrcReg = MI.getOperand(i: 0).getReg();
11146
11147 // Create new virtual register of the same class as source.
11148 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
11149 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
11150
11151 // Replace pseudo with a normal load-and-test that models the def as
11152 // well.
11153 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
11154 .addReg(RegNo: SrcReg)
11155 .setMIFlags(MI.getFlags());
11156 MI.eraseFromParent();
11157
11158 return MBB;
11159}
11160
11161MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
11162 MachineInstr &MI, MachineBasicBlock *MBB) const {
11163 MachineFunction &MF = *MBB->getParent();
11164 MachineRegisterInfo *MRI = &MF.getRegInfo();
11165 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11166 DebugLoc DL = MI.getDebugLoc();
11167 const unsigned ProbeSize = getStackProbeSize(MF);
11168 Register DstReg = MI.getOperand(i: 0).getReg();
11169 Register SizeReg = MI.getOperand(i: 2).getReg();
11170
11171 MachineBasicBlock *StartMBB = MBB;
11172 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11173 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11174 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
11175 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
11176 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
11177
11178 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
11179 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
11180
11181 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11182 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11183
11184 // LoopTestMBB
11185 // BRC TailTestMBB
11186 // # fallthrough to LoopBodyMBB
11187 StartMBB->addSuccessor(Succ: LoopTestMBB);
11188 MBB = LoopTestMBB;
11189 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
11190 .addReg(RegNo: SizeReg)
11191 .addMBB(MBB: StartMBB)
11192 .addReg(RegNo: IncReg)
11193 .addMBB(MBB: LoopBodyMBB);
11194 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
11195 .addReg(RegNo: PHIReg)
11196 .addImm(Val: ProbeSize);
11197 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11198 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
11199 .addMBB(MBB: TailTestMBB);
11200 MBB->addSuccessor(Succ: LoopBodyMBB);
11201 MBB->addSuccessor(Succ: TailTestMBB);
11202
11203 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
11204 // J LoopTestMBB
11205 MBB = LoopBodyMBB;
11206 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
11207 .addReg(RegNo: PHIReg)
11208 .addImm(Val: ProbeSize)
11209 .setOperandDead(3);
11210 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
11211 .addReg(RegNo: SystemZ::R15D)
11212 .addImm(Val: ProbeSize)
11213 .setOperandDead(3);
11214 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11215 .addReg(RegNo: SystemZ::R15D)
11216 .addReg(RegNo: SystemZ::R15D)
11217 .addImm(Val: ProbeSize - 8)
11218 .addReg(RegNo: 0)
11219 .setOperandDead(4)
11220 .setMemRefs(VolLdMMO);
11221 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
11222 MBB->addSuccessor(Succ: LoopTestMBB);
11223
11224 // TailTestMBB
11225 // BRC DoneMBB
11226 // # fallthrough to TailMBB
11227 MBB = TailTestMBB;
11228 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
11229 .addReg(RegNo: PHIReg)
11230 .addImm(Val: 0);
11231 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11232 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
11233 .addMBB(MBB: DoneMBB);
11234 MBB->addSuccessor(Succ: TailMBB);
11235 MBB->addSuccessor(Succ: DoneMBB);
11236
11237 // TailMBB
11238 // # fallthrough to DoneMBB
11239 MBB = TailMBB;
11240 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
11241 .addReg(RegNo: SystemZ::R15D)
11242 .addReg(RegNo: PHIReg)
11243 .setOperandDead(3);
11244 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11245 .addReg(RegNo: SystemZ::R15D)
11246 .addReg(RegNo: SystemZ::R15D)
11247 .addImm(Val: -8)
11248 .addReg(RegNo: PHIReg)
11249 .setOperandDead(4)
11250 .setMemRefs(VolLdMMO);
11251 MBB->addSuccessor(Succ: DoneMBB);
11252
11253 // DoneMBB
11254 MBB = DoneMBB;
11255 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
11256 .addReg(RegNo: SystemZ::R15D);
11257
11258 MI.eraseFromParent();
11259 return DoneMBB;
11260}
11261
11262SDValue SystemZTargetLowering::
11263getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11264 MachineFunction &MF = DAG.getMachineFunction();
11265 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11266 SDLoc DL(SP);
11267 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
11268 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
11269}
11270
11271// Replace a _STACKGUARD_DAG pseudo with a _STACKGUARD pseudo, adding
11272// a dead early-clobber def reg that will be used as a scratch register
11273// when the pseudo is expanded.
11274MachineBasicBlock *SystemZTargetLowering::emitStackGuardPseudo(
11275 MachineInstr &MI, MachineBasicBlock *MBB, unsigned PseudoOp) const {
11276 MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
11277 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11278 DebugLoc DL = MI.getDebugLoc();
11279 Register AddrReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11280 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PseudoOp), DestReg: AddrReg)
11281 .addFrameIndex(Idx: MI.getOperand(i: 0).getIndex())
11282 .addImm(Val: MI.getOperand(i: 1).getImm());
11283 MI.eraseFromParent();
11284 return MBB;
11285}
11286
11287MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
11288 MachineInstr &MI, MachineBasicBlock *MBB) const {
11289 switch (MI.getOpcode()) {
11290 case SystemZ::ADJCALLSTACKDOWN:
11291 case SystemZ::ADJCALLSTACKUP:
11292 return emitAdjCallStack(MI, BB: MBB);
11293
11294 case SystemZ::Select32:
11295 case SystemZ::Select64:
11296 case SystemZ::Select128:
11297 case SystemZ::SelectF32:
11298 case SystemZ::SelectF64:
11299 case SystemZ::SelectF128:
11300 case SystemZ::SelectVR32:
11301 case SystemZ::SelectVR64:
11302 case SystemZ::SelectVR128:
11303 return emitSelect(MI, MBB);
11304
11305 case SystemZ::CondStore8Mux:
11306 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
11307 case SystemZ::CondStore8MuxInv:
11308 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
11309 case SystemZ::CondStore16Mux:
11310 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
11311 case SystemZ::CondStore16MuxInv:
11312 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
11313 case SystemZ::CondStore32Mux:
11314 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
11315 case SystemZ::CondStore32MuxInv:
11316 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
11317 case SystemZ::CondStore8:
11318 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
11319 case SystemZ::CondStore8Inv:
11320 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
11321 case SystemZ::CondStore16:
11322 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
11323 case SystemZ::CondStore16Inv:
11324 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
11325 case SystemZ::CondStore32:
11326 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
11327 case SystemZ::CondStore32Inv:
11328 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
11329 case SystemZ::CondStore64:
11330 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
11331 case SystemZ::CondStore64Inv:
11332 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
11333 case SystemZ::CondStoreF32:
11334 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
11335 case SystemZ::CondStoreF32Inv:
11336 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
11337 case SystemZ::CondStoreF64:
11338 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
11339 case SystemZ::CondStoreF64Inv:
11340 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
11341
11342 case SystemZ::SCmp128Hi:
11343 return emitICmp128Hi(MI, MBB, Unsigned: false);
11344 case SystemZ::UCmp128Hi:
11345 return emitICmp128Hi(MI, MBB, Unsigned: true);
11346
11347
11348 case SystemZ::ATOMIC_SWAPW:
11349 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
11350
11351 case SystemZ::ATOMIC_LOADW_AR:
11352 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
11353 case SystemZ::ATOMIC_LOADW_AFI:
11354 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
11355
11356 case SystemZ::ATOMIC_LOADW_SR:
11357 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
11358
11359 case SystemZ::ATOMIC_LOADW_NR:
11360 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
11361 case SystemZ::ATOMIC_LOADW_NILH:
11362 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
11363
11364 case SystemZ::ATOMIC_LOADW_OR:
11365 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
11366 case SystemZ::ATOMIC_LOADW_OILH:
11367 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
11368
11369 case SystemZ::ATOMIC_LOADW_XR:
11370 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
11371 case SystemZ::ATOMIC_LOADW_XILF:
11372 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
11373
11374 case SystemZ::ATOMIC_LOADW_NRi:
11375 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
11376 case SystemZ::ATOMIC_LOADW_NILHi:
11377 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
11378
11379 case SystemZ::ATOMIC_LOADW_MIN:
11380 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11381 case SystemZ::ATOMIC_LOADW_MAX:
11382 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11383 case SystemZ::ATOMIC_LOADW_UMIN:
11384 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11385 case SystemZ::ATOMIC_LOADW_UMAX:
11386 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11387
11388 case SystemZ::ATOMIC_CMP_SWAPW:
11389 return emitAtomicCmpSwapW(MI, MBB);
11390 case SystemZ::MVCImm:
11391 case SystemZ::MVCReg:
11392 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
11393 case SystemZ::NCImm:
11394 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
11395 case SystemZ::OCImm:
11396 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
11397 case SystemZ::XCImm:
11398 case SystemZ::XCReg:
11399 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
11400 case SystemZ::CLCImm:
11401 case SystemZ::CLCReg:
11402 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
11403 case SystemZ::MemsetImmImm:
11404 case SystemZ::MemsetImmReg:
11405 case SystemZ::MemsetRegImm:
11406 case SystemZ::MemsetRegReg:
11407 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
11408 case SystemZ::MemmoveImm:
11409 return emitMemmoveImm(MI, MBB);
11410 case SystemZ::CLSTLoop:
11411 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
11412 case SystemZ::MVSTLoop:
11413 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
11414 case SystemZ::SRSTLoop:
11415 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
11416 case SystemZ::TBEGIN:
11417 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
11418 case SystemZ::TBEGIN_nofloat:
11419 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
11420 case SystemZ::TBEGINC:
11421 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
11422 case SystemZ::LTEBRCompare_Pseudo:
11423 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
11424 case SystemZ::LTDBRCompare_Pseudo:
11425 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
11426 case SystemZ::LTXBRCompare_Pseudo:
11427 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
11428
11429 case SystemZ::PROBED_ALLOCA:
11430 return emitProbedAlloca(MI, MBB);
11431 case SystemZ::EH_SjLj_SetJmp:
11432 return emitEHSjLjSetJmp(MI, MBB);
11433 case SystemZ::EH_SjLj_LongJmp:
11434 return emitEHSjLjLongJmp(MI, MBB);
11435
11436 case TargetOpcode::STACKMAP:
11437 case TargetOpcode::PATCHPOINT:
11438 return emitPatchPoint(MI, MBB);
11439
11440 case SystemZ::MOV_STACKGUARD_DAG:
11441 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::MOV_STACKGUARD);
11442
11443 case SystemZ::CMP_STACKGUARD_DAG:
11444 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::CMP_STACKGUARD);
11445
11446 default:
11447 llvm_unreachable("Unexpected instr type to insert");
11448 }
11449}
11450
11451SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11452 SelectionDAG &DAG) const {
11453 SDLoc dl(Op);
11454 /*
11455 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11456 settings:
11457 00 Round to nearest
11458 01 Round to 0
11459 10 Round to +inf
11460 11 Round to -inf
11461
11462 FLT_ROUNDS, on the other hand, expects the following:
11463 -1 Undefined
11464 0 Round to 0
11465 1 Round to nearest
11466 2 Round to +inf
11467 3 Round to -inf
11468 */
11469
11470 // Save FPC to register.
11471 SDValue Chain = Op.getOperand(i: 0);
11472 SDValue EFPC(
11473 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11474 Chain = EFPC.getValue(R: 1);
11475
11476 // Transform as necessary
11477 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11478 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11479 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11480 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11481 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11482 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11483
11484 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11485 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11486 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11487
11488 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11489}
11490
11491SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11492 SelectionDAG &DAG) const {
11493 EVT VT = Op.getValueType();
11494 Op = Op.getOperand(i: 0);
11495 EVT OpVT = Op.getValueType();
11496
11497 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11498
11499 SDLoc DL(Op);
11500
11501 // load a 0 vector for the third operand of VSUM.
11502 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11503
11504 // execute VSUM.
11505 switch (OpVT.getScalarSizeInBits()) {
11506 case 8:
11507 case 16:
11508 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11509 [[fallthrough]];
11510 case 32:
11511 case 64:
11512 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11513 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11514 break;
11515 case 128:
11516 break; // VSUM over v1i128 should not happen and would be a noop
11517 default:
11518 llvm_unreachable("Unexpected scalar size.");
11519 }
11520 // Cast to original vector type, retrieve last element.
11521 return DAG.getNode(
11522 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11523 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11524}
11525
11526static void printFunctionArgExts(const Function *F, raw_ostream &OS) {
11527 FunctionType *FT = F->getFunctionType();
11528 const AttributeList &Attrs = F->getAttributes();
11529 if (Attrs.hasRetAttrs())
11530 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11531 OS << *F->getReturnType() << " @" << F->getName() << "(";
11532 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11533 if (I)
11534 OS << ", ";
11535 OS << *FT->getParamType(i: I);
11536 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11537 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11538 if (ArgAttrs.hasAttribute(Kind: A))
11539 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11540 }
11541 if (F->isVarArg())
11542 OS << ", ...";
11543 OS << ")\n";
11544}
11545
11546bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11547 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11548 if (Itr == IsInternalCache.end())
11549 Itr = IsInternalCache
11550 .insert(x: std::pair<const Function *, bool>(
11551 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11552 .first;
11553 return Itr->second;
11554}
11555
11556bool SystemZTargetLowering::enableNarrowIntArgsVerification() const {
11557 if (!Subtarget.isTargetELF())
11558 return false;
11559
11560 if (EnableIntArgExtCheck.getNumOccurrences())
11561 return EnableIntArgExtCheck;
11562
11563#ifdef NDEBUG
11564 return false;
11565#endif
11566
11567 return getTargetMachine().Options.VerifyArgABICompliance;
11568}
11569
11570void SystemZTargetLowering::verifyNarrowIntegerArgs_Call(
11571 CallLoweringInfo &CLI, const SmallVectorImpl<ISD::OutputArg> &Outs,
11572 const Function *F, SDValue Callee) const {
11573 if (!enableNarrowIntArgsVerification())
11574 return;
11575 bool IsInternal = false;
11576 const Function *CalleeFn = nullptr;
11577 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11578 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11579 IsInternal = isInternal(Fn: CalleeFn);
11580 bool C_ABI = CLI.CallConv == CallingConv::C && !IsInternal;
11581 if (!verifyNarrowIntegerArgs(C_ABI, Outs, FunHeader: CalleeFn, /*IsRet=*/false)) {
11582 errs() << "Callee: ";
11583 if (CalleeFn != nullptr)
11584 printFunctionArgExts(F: CalleeFn, OS&: errs());
11585 else
11586 errs() << "-\n";
11587 errs() << "Caller: ";
11588 printFunctionArgExts(F, OS&: errs());
11589 if (CLI.CB)
11590 errs() << " " << *CLI.CB;
11591 llvm_unreachable("");
11592 }
11593 LLVM_DEBUG(dbgs() << "Outgoing call arguments verified as ABI compliant: ";
11594 if (CalleeFn != nullptr) printFunctionArgExts(CalleeFn, dbgs());
11595 else dbgs() << "-\n";);
11596}
11597
11598void SystemZTargetLowering::verifyNarrowIntegerArgs_Ret(
11599 CallingConv::ID CallConv, const SmallVectorImpl<ISD::OutputArg> &Outs,
11600 const Function *F) const {
11601 if (!enableNarrowIntArgsVerification())
11602 return;
11603 bool C_ABI = CallConv == CallingConv::C && !isInternal(Fn: F);
11604 if (!verifyNarrowIntegerArgs(C_ABI, Outs, FunHeader: F, /*IsRet=*/true)) {
11605 errs() << "Returning from function: ";
11606 printFunctionArgExts(F, OS&: errs());
11607 llvm_unreachable("");
11608 }
11609 LLVM_DEBUG(dbgs() << "Return argument verified as ABI compliant : ";
11610 printFunctionArgExts(F, dbgs()));
11611}
11612
11613// Verify that narrow integer arguments are extended according to the
11614// function header (calllee for a call, containing function for a ret).
11615// If C_ABI is true, an extension attribute must be present.
11616bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11617 bool C_ABI, const SmallVectorImpl<ISD::OutputArg> &Outs,
11618 const Function *FunHeader, bool IsRet) const {
11619 if (IsRet && Outs.size() != 1)
11620 return true;
11621
11622 for (unsigned i = 0; i < Outs.size(); ++i) {
11623 EVT OrigArgVT = Outs[i].ArgVT;
11624 if (!OrigArgVT.isScalarInteger() || OrigArgVT.getSizeInBits() >= 64)
11625 continue;
11626 assert(Outs[i].VT == MVT::i32 && "Unexpected integer argument VT.");
11627 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11628 unsigned OrigIdx = Outs[i].OrigArgIndex;
11629
11630 // Per the C ABI there must be an extension.
11631 if (C_ABI && !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt()) {
11632 errs() << "ERROR: (C ABI violation) missing extension attribute on "
11633 << (IsRet ? "return value" : Twine("arg ") + Twine(OrigIdx))
11634 << ".\n";
11635 return false;
11636 }
11637
11638 // Flags should reflect the attributes of the function header.
11639 if (FunHeader) {
11640 const AttributeList &FunAttrs = FunHeader->getAttributes();
11641 const AttributeSet Attrs =
11642 IsRet ? FunAttrs.getRetAttrs() : FunAttrs.getParamAttrs(ArgNo: OrigIdx);
11643 if (Attrs.hasAttribute(Kind: Attribute::SExt) && !Flags.isSExt()) {
11644 errs() << "ERROR: Missing SExt on arg " << OrigIdx << ".\n";
11645 return false;
11646 }
11647 if (Attrs.hasAttribute(Kind: Attribute::ZExt) && !Flags.isZExt()) {
11648 errs() << "ERROR: Missing ZExt on arg " << OrigIdx << ".\n";
11649 return false;
11650 }
11651 }
11652 }
11653
11654 return true;
11655}
11656
11657void SystemZTargetLowering::insertSSPDeclarations(
11658 Module &M, const LibcallLoweringInfo &Libcalls) const {
11659 StringRef GuardMode = M.getStackProtectorGuard();
11660
11661 // In the TLS case, no symbol needs to be inserted.
11662 if (GuardMode == "tls" || GuardMode.empty())
11663 return;
11664
11665 // Otherwise (in the global case), insert the appropriate global variable.
11666 TargetLowering::insertSSPDeclarations(M, Libcalls);
11667}
11668