1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZTargetMachine.h"
15#include "llvm/Analysis/AliasAnalysis.h"
16#include "llvm/CodeGen/SelectionDAGISel.h"
17#include "llvm/IR/Module.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/KnownBits.h"
20#include "llvm/Support/raw_ostream.h"
21
22using namespace llvm;
23
24#define DEBUG_TYPE "systemz-isel"
25#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
26
27namespace {
28// Used to build addressing modes.
29struct SystemZAddressingMode {
30 // The shape of the address.
31 enum AddrForm {
32 // base+displacement
33 FormBD,
34
35 // base+displacement+index for load and store operands
36 FormBDXNormal,
37
38 // base+displacement+index for load address operands
39 FormBDXLA,
40
41 // base+displacement+index+ADJDYNALLOC
42 FormBDXDynAlloc
43 };
44 AddrForm Form;
45
46 // The type of displacement. The enum names here correspond directly
47 // to the definitions in SystemZOperand.td. We could split them into
48 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
49 enum DispRange {
50 Disp12Only,
51 Disp12Pair,
52 Disp20Only,
53 Disp20Only128,
54 Disp20Pair
55 };
56 DispRange DR;
57
58 // The parts of the address. The address is equivalent to:
59 //
60 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 SDValue Base;
62 int64_t Disp;
63 SDValue Index;
64 bool IncludesDynAlloc;
65
66 SystemZAddressingMode(AddrForm form, DispRange dr)
67 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
68
69 // True if the address can have an index register.
70 bool hasIndexField() { return Form != FormBD; }
71
72 // True if the address can (and must) include ADJDYNALLOC.
73 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
74
75 void dump(const llvm::SelectionDAG *DAG) {
76 errs() << "SystemZAddressingMode " << this << '\n';
77
78 errs() << " Base ";
79 if (Base.getNode())
80 Base.getNode()->dump(G: DAG);
81 else
82 errs() << "null\n";
83
84 if (hasIndexField()) {
85 errs() << " Index ";
86 if (Index.getNode())
87 Index.getNode()->dump(G: DAG);
88 else
89 errs() << "null\n";
90 }
91
92 errs() << " Disp " << Disp;
93 if (IncludesDynAlloc)
94 errs() << " + ADJDYNALLOC";
95 errs() << '\n';
96 }
97};
98
99// Return a mask with Count low bits set.
100static uint64_t allOnes(unsigned int Count) {
101 assert(Count <= 64);
102 if (Count > 63)
103 return UINT64_MAX;
104 return (uint64_t(1) << Count) - 1;
105}
106
107// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
108// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
109// Rotate (I5). The combined operand value is effectively:
110//
111// (or (rotl Input, Rotate), ~Mask)
112//
113// for RNSBG and:
114//
115// (and (rotl Input, Rotate), Mask)
116//
117// otherwise. The output value has BitSize bits, although Input may be
118// narrower (in which case the upper bits are don't care), or wider (in which
119// case the result will be truncated as part of the operation).
120struct RxSBGOperands {
121 RxSBGOperands(unsigned Op, SDValue N)
122 : Opcode(Op), BitSize(N.getValueSizeInBits()),
123 Mask(allOnes(Count: BitSize)), Input(N), Start(64 - BitSize), End(63),
124 Rotate(0) {}
125
126 unsigned Opcode;
127 unsigned BitSize;
128 uint64_t Mask;
129 SDValue Input;
130 unsigned Start;
131 unsigned End;
132 unsigned Rotate;
133};
134
135class SystemZDAGToDAGISel : public SelectionDAGISel {
136 const SystemZSubtarget *Subtarget;
137
138 // Used by SystemZOperands.td to create integer constants.
139 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
140 return CurDAG->getTargetConstant(Val: Imm, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
141 }
142
143 const SystemZTargetMachine &getTargetMachine() const {
144 return static_cast<const SystemZTargetMachine &>(TM);
145 }
146
147 const SystemZInstrInfo *getInstrInfo() const {
148 return Subtarget->getInstrInfo();
149 }
150
151 // Try to fold more of the base or index of AM into AM, where IsBase
152 // selects between the base and index.
153 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
154
155 // Try to describe N in AM, returning true on success.
156 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
157
158 // Extract individual target operands from matched address AM.
159 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
160 SDValue &Base, SDValue &Disp) const;
161 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
162 SDValue &Base, SDValue &Disp, SDValue &Index) const;
163
164 // Try to match Addr as a FormBD address with displacement type DR.
165 // Return true on success, storing the base and displacement in
166 // Base and Disp respectively.
167 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
168 SDValue &Base, SDValue &Disp, int64_t Offset = 0) const;
169
170 // Try to match Addr as a FormBDX address with displacement type DR.
171 // Return true on success and if the result had no index. Store the
172 // base and displacement in Base and Disp respectively.
173 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
174 SDValue &Base, SDValue &Disp) const;
175
176 // Try to match Addr as a FormBDX* address of form Form with
177 // displacement type DR. Return true on success, storing the base,
178 // displacement and index in Base, Disp and Index respectively.
179 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
180 SystemZAddressingMode::DispRange DR, SDValue Addr,
181 SDValue &Base, SDValue &Disp, SDValue &Index) const;
182
183 // PC-relative address matching routines used by SystemZOperands.td.
184 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
185 if (SystemZISD::isPCREL(Opcode: Addr.getOpcode())) {
186 Target = Addr.getOperand(i: 0);
187 return true;
188 }
189 return false;
190 }
191
192 // BD matching routines used by SystemZOperands.td.
193 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
194 return selectBDAddr(DR: SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
195 }
196 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
197 return selectBDAddr(DR: SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
198 }
199 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
200 return selectBDAddr(DR: SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
201 }
202 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
203 return selectBDAddr(DR: SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
204 }
205
206 // MVI matching routines used by SystemZOperands.td.
207 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
208 return selectMVIAddr(DR: SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
209 }
210 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
211 return selectMVIAddr(DR: SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
212 }
213
214 // BDX matching routines used by SystemZOperands.td.
215 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
216 SDValue &Index) const {
217 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
218 DR: SystemZAddressingMode::Disp12Only,
219 Addr, Base, Disp, Index);
220 }
221 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
222 SDValue &Index) const {
223 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
224 DR: SystemZAddressingMode::Disp12Pair,
225 Addr, Base, Disp, Index);
226 }
227 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
228 SDValue &Index) const {
229 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXDynAlloc,
230 DR: SystemZAddressingMode::Disp12Only,
231 Addr, Base, Disp, Index);
232 }
233 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
234 SDValue &Index) const {
235 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
236 DR: SystemZAddressingMode::Disp20Only,
237 Addr, Base, Disp, Index);
238 }
239 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
240 SDValue &Index) const {
241 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
242 DR: SystemZAddressingMode::Disp20Only128,
243 Addr, Base, Disp, Index);
244 }
245 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
246 SDValue &Index) const {
247 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
248 DR: SystemZAddressingMode::Disp20Pair,
249 Addr, Base, Disp, Index);
250 }
251 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
252 SDValue &Index) const {
253 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXLA,
254 DR: SystemZAddressingMode::Disp12Pair,
255 Addr, Base, Disp, Index);
256 }
257 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
258 SDValue &Index) const {
259 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXLA,
260 DR: SystemZAddressingMode::Disp20Pair,
261 Addr, Base, Disp, Index);
262 }
263
264 // Try to match Addr as an address with a base, 12-bit displacement
265 // and index, where the index is element Elem of a vector.
266 // Return true on success, storing the base, displacement and vector
267 // in Base, Disp and Index respectively.
268 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
269 SDValue &Disp, SDValue &Index) const;
270 // Wrapper functions for LSB access on big-endian multi-byte types.
271 // Selects a Base + Displacement address and applies a fixed byte Offset.
272 // Offsets: i16 LSB = +1, i32 LSB = +3, i64 LSB = +7.
273 bool selectBDAddr12off1(SDValue Addr, SDValue &Base, SDValue &Disp) const;
274 bool selectBDAddr12off3(SDValue Addr, SDValue &Base, SDValue &Disp) const;
275 bool selectBDAddr12off7(SDValue Addr, SDValue &Base, SDValue &Disp) const;
276 bool selectBDAddr20off1(SDValue Addr, SDValue &Base, SDValue &Disp) const;
277 bool selectBDAddr20off3(SDValue Addr, SDValue &Base, SDValue &Disp) const;
278 bool selectBDAddr20off7(SDValue Addr, SDValue &Base, SDValue &Disp) const;
279
280 // Check whether (or Op (and X InsertMask)) is effectively an insertion
281 // of X into bits InsertMask of some Y != Op. Return true if so and
282 // set Op to that Y.
283 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
284
285 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
286 // Return true on success.
287 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
288
289 // Try to fold some of RxSBG.Input into other fields of RxSBG.
290 // Return true on success.
291 bool expandRxSBG(RxSBGOperands &RxSBG) const;
292
293 // Return an undefined value of type VT.
294 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
295
296 // Convert N to VT, if it isn't already.
297 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
298
299 // Try to implement AND or shift node N using RISBG with the zero flag set.
300 // Return the selected node on success, otherwise return null.
301 bool tryRISBGZero(SDNode *N);
302
303 // Try to use RISBG or Opcode to implement OR or XOR node N.
304 // Return the selected node on success, otherwise return null.
305 bool tryRxSBG(SDNode *N, unsigned Opcode);
306
307 // If Op0 is null, then Node is a constant that can be loaded using:
308 //
309 // (Opcode UpperVal LowerVal)
310 //
311 // If Op0 is nonnull, then Node can be implemented using:
312 //
313 // (Opcode (Opcode Op0 UpperVal) LowerVal)
314 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
315 uint64_t UpperVal, uint64_t LowerVal);
316
317 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
318 SDNode *Node);
319
320 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL);
321
322 // Try to use gather instruction Opcode to implement vector insertion N.
323 bool tryGather(SDNode *N, unsigned Opcode);
324
325 // Try to use scatter instruction Opcode to implement store Store.
326 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
327
328 // Change a chain of {load; op; store} of the same value into a simple op
329 // through memory of that value, if the uses of the modified value and its
330 // address are suitable.
331 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
332
333 // Return true if Load and Store are loads and stores of the same size
334 // and are guaranteed not to overlap. Such operations can be implemented
335 // using block (SS-format) instructions.
336 //
337 // Partial overlap would lead to incorrect code, since the block operations
338 // are logically bytewise, even though they have a fast path for the
339 // non-overlapping case. We also need to avoid full overlap (i.e. two
340 // addresses that might be equal at run time) because although that case
341 // would be handled correctly, it might be implemented by millicode.
342 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
343
344 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
345 // from Y to X.
346 bool storeLoadCanUseMVC(SDNode *N) const;
347
348 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
349 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
350 // to X.
351 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
352
353 // Return true if N (a load or a store) fullfills the alignment
354 // requirements for a PC-relative access.
355 bool storeLoadIsAligned(SDNode *N) const;
356
357 // Return the load extension type of a load or atomic load.
358 ISD::LoadExtType getLoadExtType(SDNode *N) const;
359
360 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
361 SDValue expandSelectBoolean(SDNode *Node);
362
363 // Return true if the flags of N and the subtarget allows for
364 // reassociation, in which case a reg/reg opcode is needed as input to the
365 // MachineCombiner.
366 bool shouldSelectForReassoc(SDNode *N) const;
367
368public:
369 SystemZDAGToDAGISel() = delete;
370
371 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
372 : SelectionDAGISel(TM, OptLevel) {}
373
374 bool runOnMachineFunction(MachineFunction &MF) override {
375 const Function &F = MF.getFunction();
376 if (F.getFnAttribute(Kind: "fentry-call").getValueAsString() != "true") {
377 if (F.hasFnAttribute(Kind: "mnop-mcount"))
378 report_fatal_error(reason: "mnop-mcount only supported with fentry-call");
379 if (F.hasFnAttribute(Kind: "mrecord-mcount"))
380 report_fatal_error(reason: "mrecord-mcount only supported with fentry-call");
381 }
382 if (F.getParent()->getStackProtectorGuard() != "global") {
383 if (F.getParent()->hasStackProtectorGuardRecord())
384 report_fatal_error(reason: "mstack-protector-guard-record only supported with "
385 "mstack-protector-guard=global");
386 }
387 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
388 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
389 }
390
391 // Override SelectionDAGISel.
392 void Select(SDNode *Node) override;
393 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
394 InlineAsm::ConstraintCode ConstraintID,
395 std::vector<SDValue> &OutOps) override;
396 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
397 void PreprocessISelDAG() override;
398
399 // Include the pieces autogenerated from the target description.
400 #include "SystemZGenDAGISel.inc"
401};
402
403class SystemZDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
404public:
405 static char ID;
406 explicit SystemZDAGToDAGISelLegacy(SystemZTargetMachine &TM,
407 CodeGenOptLevel OptLevel)
408 : SelectionDAGISelLegacy(
409 ID, std::make_unique<SystemZDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
410};
411} // end anonymous namespace
412
413char SystemZDAGToDAGISelLegacy::ID = 0;
414
415INITIALIZE_PASS(SystemZDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
416
417FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
418 CodeGenOptLevel OptLevel) {
419 return new SystemZDAGToDAGISelLegacy(TM, OptLevel);
420}
421
422// Return true if Val should be selected as a displacement for an address
423// with range DR. Here we're interested in the range of both the instruction
424// described by DR and of any pairing instruction.
425static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
426 switch (DR) {
427 case SystemZAddressingMode::Disp12Only:
428 return isUInt<12>(x: Val);
429
430 case SystemZAddressingMode::Disp12Pair:
431 case SystemZAddressingMode::Disp20Only:
432 case SystemZAddressingMode::Disp20Pair:
433 return isInt<20>(x: Val);
434
435 case SystemZAddressingMode::Disp20Only128:
436 return isInt<20>(x: Val) && isInt<20>(x: Val + 8);
437 }
438 llvm_unreachable("Unhandled displacement range");
439}
440
441// Change the base or index in AM to Value, where IsBase selects
442// between the base and index.
443static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
444 SDValue Value) {
445 if (IsBase)
446 AM.Base = Value;
447 else
448 AM.Index = Value;
449}
450
451// The base or index of AM is equivalent to Value + ADJDYNALLOC,
452// where IsBase selects between the base and index. Try to fold the
453// ADJDYNALLOC into AM.
454static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
455 SDValue Value) {
456 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
457 changeComponent(AM, IsBase, Value);
458 AM.IncludesDynAlloc = true;
459 return true;
460 }
461 return false;
462}
463
464// The base of AM is equivalent to Base + Index. Try to use Index as
465// the index register.
466static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
467 SDValue Index) {
468 if (AM.hasIndexField() && !AM.Index.getNode()) {
469 AM.Base = Base;
470 AM.Index = Index;
471 return true;
472 }
473 return false;
474}
475
476// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
477// between the base and index. Try to fold Op1 into AM's displacement.
478static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
479 SDValue Op0, uint64_t Op1) {
480 // First try adjusting the displacement.
481 int64_t TestDisp = AM.Disp + Op1;
482 if (selectDisp(DR: AM.DR, Val: TestDisp)) {
483 changeComponent(AM, IsBase, Value: Op0);
484 AM.Disp = TestDisp;
485 return true;
486 }
487
488 // We could consider forcing the displacement into a register and
489 // using it as an index, but it would need to be carefully tuned.
490 return false;
491}
492
493bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
494 bool IsBase) const {
495 SDValue N = IsBase ? AM.Base : AM.Index;
496 unsigned Opcode = N.getOpcode();
497 // Look through no-op truncations.
498 if (Opcode == ISD::TRUNCATE && N.getOperand(i: 0).getValueSizeInBits() <= 64) {
499 N = N.getOperand(i: 0);
500 Opcode = N.getOpcode();
501 }
502 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(Op: N)) {
503 SDValue Op0 = N.getOperand(i: 0);
504 SDValue Op1 = N.getOperand(i: 1);
505
506 unsigned Op0Code = Op0->getOpcode();
507 unsigned Op1Code = Op1->getOpcode();
508
509 if (Op0Code == SystemZISD::ADJDYNALLOC)
510 return expandAdjDynAlloc(AM, IsBase, Value: Op1);
511 if (Op1Code == SystemZISD::ADJDYNALLOC)
512 return expandAdjDynAlloc(AM, IsBase, Value: Op0);
513
514 if (Op0Code == ISD::Constant)
515 return expandDisp(AM, IsBase, Op0: Op1,
516 Op1: cast<ConstantSDNode>(Val&: Op0)->getSExtValue());
517 if (Op1Code == ISD::Constant)
518 return expandDisp(AM, IsBase, Op0,
519 Op1: cast<ConstantSDNode>(Val&: Op1)->getSExtValue());
520
521 if (IsBase && expandIndex(AM, Base: Op0, Index: Op1))
522 return true;
523 }
524 if (Opcode == SystemZISD::PCREL_OFFSET) {
525 SDValue Full = N.getOperand(i: 0);
526 SDValue Base = N.getOperand(i: 1);
527 SDValue Anchor = Base.getOperand(i: 0);
528 uint64_t Offset = (cast<GlobalAddressSDNode>(Val&: Full)->getOffset() -
529 cast<GlobalAddressSDNode>(Val&: Anchor)->getOffset());
530 return expandDisp(AM, IsBase, Op0: Base, Op1: Offset);
531 }
532 return false;
533}
534
535// Return true if an instruction with displacement range DR should be
536// used for displacement value Val. selectDisp(DR, Val) must already hold.
537static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
538 assert(selectDisp(DR, Val) && "Invalid displacement");
539 switch (DR) {
540 case SystemZAddressingMode::Disp12Only:
541 case SystemZAddressingMode::Disp20Only:
542 case SystemZAddressingMode::Disp20Only128:
543 return true;
544
545 case SystemZAddressingMode::Disp12Pair:
546 // Use the other instruction if the displacement is too large.
547 return isUInt<12>(x: Val);
548
549 case SystemZAddressingMode::Disp20Pair:
550 // Use the other instruction if the displacement is small enough.
551 return !isUInt<12>(x: Val);
552 }
553 llvm_unreachable("Unhandled displacement range");
554}
555
556// Return true if Base + Disp + Index should be performed by LA(Y).
557static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
558 // Don't use LA(Y) for constants.
559 if (!Base)
560 return false;
561
562 // Always use LA(Y) for frame addresses, since we know that the destination
563 // register is almost always (perhaps always) going to be different from
564 // the frame register.
565 if (Base->getOpcode() == ISD::FrameIndex)
566 return true;
567
568 if (Disp) {
569 // Always use LA(Y) if there is a base, displacement and index.
570 if (Index)
571 return true;
572
573 // Always use LA if the displacement is small enough. It should always
574 // be no worse than AGHI (and better if it avoids a move).
575 if (isUInt<12>(x: Disp))
576 return true;
577
578 // For similar reasons, always use LAY if the constant is too big for AGHI.
579 // LAY should be no worse than AGFI.
580 if (!isInt<16>(x: Disp))
581 return true;
582 } else {
583 // Don't use LA for plain registers.
584 if (!Index)
585 return false;
586
587 // Don't use LA for plain addition if the index operand is only used
588 // once. It should be a natural two-operand addition in that case.
589 if (Index->hasOneUse())
590 return false;
591
592 // Prefer addition if the second operation is sign-extended, in the
593 // hope of using AGF.
594 unsigned IndexOpcode = Index->getOpcode();
595 if (IndexOpcode == ISD::SIGN_EXTEND ||
596 IndexOpcode == ISD::SIGN_EXTEND_INREG)
597 return false;
598 }
599
600 // Don't use LA for two-operand addition if either operand is only
601 // used once. The addition instructions are better in that case.
602 if (Base->hasOneUse())
603 return false;
604
605 return true;
606}
607
608// Return true if Addr is suitable for AM, updating AM if so.
609bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
610 SystemZAddressingMode &AM) const {
611 // Start out assuming that the address will need to be loaded separately,
612 // then try to extend it as much as we can.
613 AM.Base = Addr;
614
615 // First try treating the address as a constant.
616 if (Addr.getOpcode() == ISD::Constant &&
617 expandDisp(AM, IsBase: true, Op0: SDValue(),
618 Op1: cast<ConstantSDNode>(Val&: Addr)->getSExtValue()))
619 ;
620 // Also see if it's a bare ADJDYNALLOC.
621 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
622 expandAdjDynAlloc(AM, IsBase: true, Value: SDValue()))
623 ;
624 else
625 // Otherwise try expanding each component.
626 while (expandAddress(AM, IsBase: true) ||
627 (AM.Index.getNode() && expandAddress(AM, IsBase: false)))
628 continue;
629
630 // Reject cases where it isn't profitable to use LA(Y).
631 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
632 !shouldUseLA(Base: AM.Base.getNode(), Disp: AM.Disp, Index: AM.Index.getNode()))
633 return false;
634
635 // Reject cases where the other instruction in a pair should be used.
636 if (!isValidDisp(DR: AM.DR, Val: AM.Disp))
637 return false;
638
639 // Make sure that ADJDYNALLOC is included where necessary.
640 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
641 return false;
642
643 LLVM_DEBUG(AM.dump(CurDAG));
644 return true;
645}
646
647// Insert a node into the DAG at least before Pos. This will reposition
648// the node as needed, and will assign it a node ID that is <= Pos's ID.
649// Note that this does *not* preserve the uniqueness of node IDs!
650// The selection DAG must no longer depend on their uniqueness when this
651// function is used.
652static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
653 if (N->getNodeId() == -1 ||
654 (SelectionDAGISel::getUninvalidatedNodeId(N: N.getNode()) >
655 SelectionDAGISel::getUninvalidatedNodeId(N: Pos))) {
656 DAG->RepositionNode(Position: Pos->getIterator(), N: N.getNode());
657 // Mark Node as invalid for pruning as after this it may be a successor to a
658 // selected node but otherwise be in the same position of Pos.
659 // Conservatively mark it with the same -abs(Id) to assure node id
660 // invariant is preserved.
661 N->setNodeId(Pos->getNodeId());
662 SelectionDAGISel::InvalidateNodeId(N: N.getNode());
663 }
664}
665
666void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
667 EVT VT, SDValue &Base,
668 SDValue &Disp) const {
669 Base = AM.Base;
670 if (!Base.getNode())
671 // Register 0 means "no base". This is mostly useful for shifts.
672 Base = CurDAG->getRegister(Reg: 0, VT);
673 else if (Base.getOpcode() == ISD::FrameIndex) {
674 // Lower a FrameIndex to a TargetFrameIndex.
675 int64_t FrameIndex = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
676 Base = CurDAG->getTargetFrameIndex(FI: FrameIndex, VT);
677 } else if (Base.getValueType() != VT) {
678 // Truncate values from i64 to i32, for shifts.
679 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
680 "Unexpected truncation");
681 SDLoc DL(Base);
682 SDValue Trunc = CurDAG->getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Base);
683 insertDAGNode(DAG: CurDAG, Pos: Base.getNode(), N: Trunc);
684 Base = Trunc;
685 }
686
687 // Lower the displacement to a TargetConstant.
688 Disp = CurDAG->getSignedTargetConstant(Val: AM.Disp, DL: SDLoc(Base), VT);
689}
690
691void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
692 EVT VT, SDValue &Base,
693 SDValue &Disp,
694 SDValue &Index) const {
695 getAddressOperands(AM, VT, Base, Disp);
696
697 Index = AM.Index;
698 if (!Index.getNode())
699 // Register 0 means "no index".
700 Index = CurDAG->getRegister(Reg: 0, VT);
701}
702
703bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
704 SDValue Addr, SDValue &Base,
705 SDValue &Disp, int64_t Offset) const {
706 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
707 AM.Disp += Offset;
708 if (!selectAddress(Addr, AM))
709 return false;
710
711 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp);
712 return true;
713}
714
715bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
716 SDValue Addr, SDValue &Base,
717 SDValue &Disp) const {
718 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
719 if (!selectAddress(Addr, AM) || AM.Index.getNode())
720 return false;
721
722 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp);
723 return true;
724}
725
726bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
727 SystemZAddressingMode::DispRange DR,
728 SDValue Addr, SDValue &Base,
729 SDValue &Disp, SDValue &Index) const {
730 SystemZAddressingMode AM(Form, DR);
731 if (!selectAddress(Addr, AM))
732 return false;
733
734 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp, Index);
735 return true;
736}
737
738bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
739 SDValue &Base,
740 SDValue &Disp,
741 SDValue &Index) const {
742 SDValue Regs[2];
743 if (selectBDXAddr12Only(Addr, Base&: Regs[0], Disp, Index&: Regs[1]) &&
744 Regs[0].getNode() && Regs[1].getNode()) {
745 for (unsigned int I = 0; I < 2; ++I) {
746 Base = Regs[I];
747 Index = Regs[1 - I];
748 // We can't tell here whether the index vector has the right type
749 // for the access; the caller needs to do that instead.
750 if (Index.getOpcode() == ISD::ZERO_EXTEND)
751 Index = Index.getOperand(i: 0);
752 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
753 Index.getOperand(i: 1) == Elem) {
754 Index = Index.getOperand(i: 0);
755 return true;
756 }
757 }
758 }
759 return false;
760}
761
762bool SystemZDAGToDAGISel::selectBDAddr12off1(SDValue A, SDValue &B,
763 SDValue &D) const {
764 return selectBDAddr(DR: SystemZAddressingMode::Disp12Pair, Addr: A, Base&: B, Disp&: D, Offset: 1);
765}
766bool SystemZDAGToDAGISel::selectBDAddr12off3(SDValue A, SDValue &B,
767 SDValue &D) const {
768 return selectBDAddr(DR: SystemZAddressingMode::Disp12Pair, Addr: A, Base&: B, Disp&: D, Offset: 3);
769}
770bool SystemZDAGToDAGISel::selectBDAddr12off7(SDValue A, SDValue &B,
771 SDValue &D) const {
772 return selectBDAddr(DR: SystemZAddressingMode::Disp12Pair, Addr: A, Base&: B, Disp&: D, Offset: 7);
773}
774
775bool SystemZDAGToDAGISel::selectBDAddr20off1(SDValue A, SDValue &B,
776 SDValue &D) const {
777 return selectBDAddr(DR: SystemZAddressingMode::Disp20Pair, Addr: A, Base&: B, Disp&: D, Offset: 1);
778}
779bool SystemZDAGToDAGISel::selectBDAddr20off3(SDValue A, SDValue &B,
780 SDValue &D) const {
781 return selectBDAddr(DR: SystemZAddressingMode::Disp20Pair, Addr: A, Base&: B, Disp&: D, Offset: 3);
782}
783bool SystemZDAGToDAGISel::selectBDAddr20off7(SDValue A, SDValue &B,
784 SDValue &D) const {
785 return selectBDAddr(DR: SystemZAddressingMode::Disp20Pair, Addr: A, Base&: B, Disp&: D, Offset: 7);
786}
787
788bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
789 uint64_t InsertMask) const {
790 // We're only interested in cases where the insertion is into some operand
791 // of Op, rather than into Op itself. The only useful case is an AND.
792 if (Op.getOpcode() != ISD::AND)
793 return false;
794
795 // We need a constant mask.
796 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode());
797 if (!MaskNode)
798 return false;
799
800 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
801 uint64_t AndMask = MaskNode->getZExtValue();
802 if (InsertMask & AndMask)
803 return false;
804
805 // It's only an insertion if all bits are covered or are known to be zero.
806 // The inner check covers all cases but is more expensive.
807 uint64_t Used = allOnes(Count: Op.getValueSizeInBits());
808 if (Used != (AndMask | InsertMask)) {
809 KnownBits Known = CurDAG->computeKnownBits(Op: Op.getOperand(i: 0));
810 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
811 return false;
812 }
813
814 Op = Op.getOperand(i: 0);
815 return true;
816}
817
818bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
819 uint64_t Mask) const {
820 const SystemZInstrInfo *TII = getInstrInfo();
821 if (RxSBG.Rotate != 0)
822 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
823 Mask &= RxSBG.Mask;
824 if (TII->isRxSBGMask(Mask, BitSize: RxSBG.BitSize, Start&: RxSBG.Start, End&: RxSBG.End)) {
825 RxSBG.Mask = Mask;
826 return true;
827 }
828 return false;
829}
830
831// Return true if any bits of (RxSBG.Input & Mask) are significant.
832static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
833 // Rotate the mask in the same way as RxSBG.Input is rotated.
834 if (RxSBG.Rotate != 0)
835 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
836 return (Mask & RxSBG.Mask) != 0;
837}
838
839bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
840 SDValue N = RxSBG.Input;
841 unsigned Opcode = N.getOpcode();
842 switch (Opcode) {
843 case ISD::TRUNCATE: {
844 if (RxSBG.Opcode == SystemZ::RNSBG)
845 return false;
846 if (N.getOperand(i: 0).getValueSizeInBits() > 64)
847 return false;
848 uint64_t BitSize = N.getValueSizeInBits();
849 uint64_t Mask = allOnes(Count: BitSize);
850 if (!refineRxSBGMask(RxSBG, Mask))
851 return false;
852 RxSBG.Input = N.getOperand(i: 0);
853 return true;
854 }
855 case ISD::AND: {
856 if (RxSBG.Opcode == SystemZ::RNSBG)
857 return false;
858
859 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
860 if (!MaskNode)
861 return false;
862
863 SDValue Input = N.getOperand(i: 0);
864 uint64_t Mask = MaskNode->getZExtValue();
865 if (!refineRxSBGMask(RxSBG, Mask)) {
866 // If some bits of Input are already known zeros, those bits will have
867 // been removed from the mask. See if adding them back in makes the
868 // mask suitable.
869 KnownBits Known = CurDAG->computeKnownBits(Op: Input);
870 Mask |= Known.Zero.getZExtValue();
871 if (!refineRxSBGMask(RxSBG, Mask))
872 return false;
873 }
874 RxSBG.Input = Input;
875 return true;
876 }
877
878 case ISD::OR: {
879 if (RxSBG.Opcode != SystemZ::RNSBG)
880 return false;
881
882 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
883 if (!MaskNode)
884 return false;
885
886 SDValue Input = N.getOperand(i: 0);
887 uint64_t Mask = ~MaskNode->getZExtValue();
888 if (!refineRxSBGMask(RxSBG, Mask)) {
889 // If some bits of Input are already known ones, those bits will have
890 // been removed from the mask. See if adding them back in makes the
891 // mask suitable.
892 KnownBits Known = CurDAG->computeKnownBits(Op: Input);
893 Mask &= ~Known.One.getZExtValue();
894 if (!refineRxSBGMask(RxSBG, Mask))
895 return false;
896 }
897 RxSBG.Input = Input;
898 return true;
899 }
900
901 case ISD::ROTL: {
902 // Any 64-bit rotate left can be merged into the RxSBG.
903 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
904 return false;
905 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
906 if (!CountNode)
907 return false;
908
909 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
910 RxSBG.Input = N.getOperand(i: 0);
911 return true;
912 }
913
914 case ISD::ANY_EXTEND:
915 // Bits above the extended operand are don't-care.
916 RxSBG.Input = N.getOperand(i: 0);
917 return true;
918
919 case ISD::ZERO_EXTEND:
920 if (RxSBG.Opcode != SystemZ::RNSBG) {
921 // Restrict the mask to the extended operand.
922 unsigned InnerBitSize = N.getOperand(i: 0).getValueSizeInBits();
923 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: InnerBitSize)))
924 return false;
925
926 RxSBG.Input = N.getOperand(i: 0);
927 return true;
928 }
929 [[fallthrough]];
930
931 case ISD::SIGN_EXTEND: {
932 // Check that the extension bits are don't-care (i.e. are masked out
933 // by the final mask).
934 unsigned BitSize = N.getValueSizeInBits();
935 unsigned InnerBitSize = N.getOperand(i: 0).getValueSizeInBits();
936 if (maskMatters(RxSBG, Mask: allOnes(Count: BitSize) - allOnes(Count: InnerBitSize))) {
937 // In the case where only the sign bit is active, increase Rotate with
938 // the extension width.
939 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
940 RxSBG.Rotate += (BitSize - InnerBitSize);
941 else
942 return false;
943 }
944
945 RxSBG.Input = N.getOperand(i: 0);
946 return true;
947 }
948
949 case ISD::SHL: {
950 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
951 if (!CountNode)
952 return false;
953
954 uint64_t Count = CountNode->getZExtValue();
955 unsigned BitSize = N.getValueSizeInBits();
956 if (Count < 1 || Count >= BitSize)
957 return false;
958
959 if (RxSBG.Opcode == SystemZ::RNSBG) {
960 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
961 // count bits from RxSBG.Input are ignored.
962 if (maskMatters(RxSBG, Mask: allOnes(Count)))
963 return false;
964 } else {
965 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
966 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: BitSize - Count) << Count))
967 return false;
968 }
969
970 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
971 RxSBG.Input = N.getOperand(i: 0);
972 return true;
973 }
974
975 case ISD::SRL:
976 case ISD::SRA: {
977 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
978 if (!CountNode)
979 return false;
980
981 uint64_t Count = CountNode->getZExtValue();
982 unsigned BitSize = N.getValueSizeInBits();
983 if (Count < 1 || Count >= BitSize)
984 return false;
985
986 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
987 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
988 // count bits from RxSBG.Input are ignored.
989 if (maskMatters(RxSBG, Mask: allOnes(Count) << (BitSize - Count)))
990 return false;
991 } else {
992 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
993 // which is similar to SLL above.
994 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: BitSize - Count)))
995 return false;
996 }
997
998 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
999 RxSBG.Input = N.getOperand(i: 0);
1000 return true;
1001 }
1002 default:
1003 return false;
1004 }
1005}
1006
1007SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
1008 SDNode *N = CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
1009 return SDValue(N, 0);
1010}
1011
1012SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
1013 SDValue N) const {
1014 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
1015 return CurDAG->getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32,
1016 DL, VT, Operand: getUNDEF(DL, VT: MVT::i64), Subreg: N);
1017 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
1018 return CurDAG->getTargetExtractSubreg(SRIdx: SystemZ::subreg_l32, DL, VT, Operand: N);
1019 assert(N.getValueType() == VT && "Unexpected value types");
1020 return N;
1021}
1022
1023bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
1024 SDLoc DL(N);
1025 EVT VT = N->getValueType(ResNo: 0);
1026 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1027 return false;
1028 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
1029 unsigned Count = 0;
1030 while (expandRxSBG(RxSBG&: RISBG))
1031 // The widening or narrowing is expected to be free.
1032 // Counting widening or narrowing as a saved operation will result in
1033 // preferring an R*SBG over a simple shift/logical instruction.
1034 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
1035 RISBG.Input.getOpcode() != ISD::TRUNCATE)
1036 Count += 1;
1037 if (Count == 0 || isa<ConstantSDNode>(Val: RISBG.Input))
1038 return false;
1039
1040 // Prefer to use normal shift instructions over RISBG, since they can handle
1041 // all cases and are sometimes shorter.
1042 if (Count == 1 && N->getOpcode() != ISD::AND)
1043 return false;
1044
1045 // Prefer LOAD LOGICAL INDEXED ADDRESS over RISBG in the case where we
1046 // can use its displacement to pull in an addition.
1047 if (Subtarget->hasMiscellaneousExtensions4() &&
1048 RISBG.Rotate >= 1 && RISBG.Rotate <= 4 &&
1049 RISBG.Mask == (((uint64_t)1 << 32) - 1) << RISBG.Rotate &&
1050 RISBG.Input.getOpcode() == ISD::ADD)
1051 if (auto *C = dyn_cast<ConstantSDNode>(Val: RISBG.Input.getOperand(i: 1)))
1052 if (isInt<20>(x: C->getSExtValue()))
1053 return false;
1054
1055 // Prefer register extensions like LLC over RISBG. Also prefer to start
1056 // out with normal ANDs if one instruction would be enough. We can convert
1057 // these ANDs into an RISBG later if a three-address instruction is useful.
1058 if (RISBG.Rotate == 0) {
1059 bool PreferAnd = false;
1060 // Prefer AND for any 32-bit and-immediate operation.
1061 if (VT == MVT::i32)
1062 PreferAnd = true;
1063 // As well as for any 64-bit operation that can be implemented via LLC(R),
1064 // LLH(R), LLGT(R), or one of the and-immediate instructions.
1065 else if (RISBG.Mask == 0xff ||
1066 RISBG.Mask == 0xffff ||
1067 RISBG.Mask == 0x7fffffff ||
1068 SystemZ::isImmLF(Val: ~RISBG.Mask) ||
1069 SystemZ::isImmHF(Val: ~RISBG.Mask))
1070 PreferAnd = true;
1071 // And likewise for the LLZRGF instruction, which doesn't have a register
1072 // to register version.
1073 else if (auto *Load = dyn_cast<LoadSDNode>(Val&: RISBG.Input)) {
1074 if (Load->getMemoryVT() == MVT::i32 &&
1075 (Load->getExtensionType() == ISD::EXTLOAD ||
1076 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1077 RISBG.Mask == 0xffffff00 &&
1078 Subtarget->hasLoadAndZeroRightmostByte())
1079 PreferAnd = true;
1080 }
1081 if (PreferAnd) {
1082 // Replace the current node with an AND. Note that the current node
1083 // might already be that same AND, in which case it is already CSE'd
1084 // with it, and we must not call ReplaceNode.
1085 SDValue In = convertTo(DL, VT, N: RISBG.Input);
1086 SDValue Mask = CurDAG->getConstant(Val: RISBG.Mask, DL, VT);
1087 SDValue New = CurDAG->getNode(Opcode: ISD::AND, DL, VT, N1: In, N2: Mask);
1088 if (N != New.getNode()) {
1089 insertDAGNode(DAG: CurDAG, Pos: N, N: Mask);
1090 insertDAGNode(DAG: CurDAG, Pos: N, N: New);
1091 ReplaceNode(F: N, T: New.getNode());
1092 N = New.getNode();
1093 }
1094 // Now, select the machine opcode to implement this operation.
1095 if (!N->isMachineOpcode())
1096 SelectCode(N);
1097 return true;
1098 }
1099 }
1100
1101 unsigned Opcode = SystemZ::RISBG;
1102 // Prefer RISBGN if available, since it does not clobber CC.
1103 if (Subtarget->hasMiscellaneousExtensions())
1104 Opcode = SystemZ::RISBGN;
1105 EVT OpcodeVT = MVT::i64;
1106 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1107 // We can only use the 32-bit instructions if all source bits are
1108 // in the low 32 bits without wrapping, both after rotation (because
1109 // of the smaller range for Start and End) and before rotation
1110 // (because the input value is truncated).
1111 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1112 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1113 ((RISBG.End + RISBG.Rotate) & 63) >=
1114 ((RISBG.Start + RISBG.Rotate) & 63)) {
1115 Opcode = SystemZ::RISBMux;
1116 OpcodeVT = MVT::i32;
1117 RISBG.Start &= 31;
1118 RISBG.End &= 31;
1119 }
1120 SDValue Ops[5] = {
1121 getUNDEF(DL, VT: OpcodeVT),
1122 convertTo(DL, VT: OpcodeVT, N: RISBG.Input),
1123 CurDAG->getTargetConstant(Val: RISBG.Start, DL, VT: MVT::i32),
1124 CurDAG->getTargetConstant(Val: RISBG.End | 128, DL, VT: MVT::i32),
1125 CurDAG->getTargetConstant(Val: RISBG.Rotate, DL, VT: MVT::i32)
1126 };
1127 SDValue New = convertTo(
1128 DL, VT, N: SDValue(CurDAG->getMachineNode(Opcode, dl: DL, VT: OpcodeVT, Ops), 0));
1129 ReplaceNode(F: N, T: New.getNode());
1130 return true;
1131}
1132
1133bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1134 SDLoc DL(N);
1135 EVT VT = N->getValueType(ResNo: 0);
1136 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1137 return false;
1138 // Try treating each operand of N as the second operand of the RxSBG
1139 // and see which goes deepest.
1140 RxSBGOperands RxSBG[] = {
1141 RxSBGOperands(Opcode, N->getOperand(Num: 0)),
1142 RxSBGOperands(Opcode, N->getOperand(Num: 1))
1143 };
1144 unsigned Count[] = { 0, 0 };
1145 for (unsigned I = 0; I < 2; ++I)
1146 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG&: RxSBG[I]))
1147 // In cases of multiple users it seems better to keep the simple
1148 // instruction as they are one cycle faster, and it also helps in cases
1149 // where both inputs share a common node.
1150 // The widening or narrowing is expected to be free. Counting widening
1151 // or narrowing as a saved operation will result in preferring an R*SBG
1152 // over a simple shift/logical instruction.
1153 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1154 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1155 Count[I] += 1;
1156
1157 // Do nothing if neither operand is suitable.
1158 if (Count[0] == 0 && Count[1] == 0)
1159 return false;
1160
1161 // Pick the deepest second operand.
1162 unsigned I = Count[0] > Count[1] ? 0 : 1;
1163 SDValue Op0 = N->getOperand(Num: I ^ 1);
1164
1165 // Prefer IC for character insertions from memory.
1166 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1167 if (auto *Load = dyn_cast<LoadSDNode>(Val: Op0.getNode()))
1168 if (Load->getMemoryVT() == MVT::i8)
1169 return false;
1170
1171 // See whether we can avoid an AND in the first operand by converting
1172 // ROSBG to RISBG.
1173 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op&: Op0, InsertMask: RxSBG[I].Mask)) {
1174 Opcode = SystemZ::RISBG;
1175 // Prefer RISBGN if available, since it does not clobber CC.
1176 if (Subtarget->hasMiscellaneousExtensions())
1177 Opcode = SystemZ::RISBGN;
1178 }
1179
1180 SDValue Ops[5] = {
1181 convertTo(DL, VT: MVT::i64, N: Op0),
1182 convertTo(DL, VT: MVT::i64, N: RxSBG[I].Input),
1183 CurDAG->getTargetConstant(Val: RxSBG[I].Start, DL, VT: MVT::i32),
1184 CurDAG->getTargetConstant(Val: RxSBG[I].End, DL, VT: MVT::i32),
1185 CurDAG->getTargetConstant(Val: RxSBG[I].Rotate, DL, VT: MVT::i32)
1186 };
1187 SDValue New = convertTo(
1188 DL, VT, N: SDValue(CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::i64, Ops), 0));
1189 ReplaceNode(F: N, T: New.getNode());
1190 return true;
1191}
1192
1193void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1194 SDValue Op0, uint64_t UpperVal,
1195 uint64_t LowerVal) {
1196 EVT VT = Node->getValueType(ResNo: 0);
1197 SDLoc DL(Node);
1198 SDValue Upper = CurDAG->getConstant(Val: UpperVal, DL, VT);
1199 if (Op0.getNode())
1200 Upper = CurDAG->getNode(Opcode, DL, VT, N1: Op0, N2: Upper);
1201
1202 {
1203 // When we haven't passed in Op0, Upper will be a constant. In order to
1204 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1205 // SelectCode first and end up with an opaque machine node. This means that
1206 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1207 // SelectCode.
1208 //
1209 // Note that in the case where Op0 is passed in we could just call
1210 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1211 // the handle at all, but it's fine to do it here.
1212 //
1213 // TODO: This is a pretty hacky way to do this. Can we do something that
1214 // doesn't require a two paragraph explanation?
1215 HandleSDNode Handle(Upper);
1216 SelectCode(N: Upper.getNode());
1217 Upper = Handle.getValue();
1218 }
1219
1220 SDValue Lower = CurDAG->getConstant(Val: LowerVal, DL, VT);
1221 SDValue Or = CurDAG->getNode(Opcode, DL, VT, N1: Upper, N2: Lower);
1222
1223 ReplaceNode(F: Node, T: Or.getNode());
1224
1225 SelectCode(N: Or.getNode());
1226}
1227
1228void SystemZDAGToDAGISel::loadVectorConstant(
1229 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1230 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1231 VCI.Opcode == SystemZISD::REPLICATE ||
1232 VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1233 "Bad opcode!");
1234 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1235 EVT VT = Node->getValueType(ResNo: 0);
1236 SDLoc DL(Node);
1237 SmallVector<SDValue, 2> Ops;
1238 for (unsigned OpVal : VCI.OpVals)
1239 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: OpVal, DL, VT: MVT::i32));
1240 SDValue Op = CurDAG->getNode(Opcode: VCI.Opcode, DL, VT: VCI.VecVT, Ops);
1241
1242 if (VCI.VecVT == VT.getSimpleVT())
1243 ReplaceNode(F: Node, T: Op.getNode());
1244 else if (VT.getSizeInBits() == 128) {
1245 SDValue BitCast = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1246 ReplaceNode(F: Node, T: BitCast.getNode());
1247 SelectCode(N: BitCast.getNode());
1248 } else { // half, float or double
1249 unsigned SubRegIdx = (VT.getSizeInBits() == 16 ? SystemZ::subreg_h16
1250 : VT.getSizeInBits() == 32 ? SystemZ::subreg_h32
1251 : SystemZ::subreg_h64);
1252 ReplaceNode(
1253 F: Node, T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: Op).getNode());
1254 }
1255 SelectCode(N: Op.getNode());
1256}
1257
1258SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {
1259 SDNode *ResNode;
1260 assert (VT.getSizeInBits() == 128);
1261
1262 SDValue CP = CurDAG->getTargetConstantPool(
1263 C: ConstantInt::get(Ty: Type::getInt128Ty(C&: *CurDAG->getContext()), V: Val),
1264 VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1265
1266 EVT PtrVT = CP.getValueType();
1267 SDValue Ops[] = {
1268 SDValue(CurDAG->getMachineNode(Opcode: SystemZ::LARL, dl: DL, VT: PtrVT, Op1: CP), 0),
1269 CurDAG->getTargetConstant(Val: 0, DL, VT: PtrVT),
1270 CurDAG->getRegister(Reg: 0, VT: PtrVT),
1271 CurDAG->getEntryNode()
1272 };
1273 ResNode = CurDAG->getMachineNode(Opcode: SystemZ::VL, dl: DL, VT1: VT, VT2: MVT::Other, Ops);
1274
1275 // Annotate ResNode with memory operand information so that MachineInstr
1276 // queries work properly. This e.g. gives the register allocation the
1277 // required information for rematerialization.
1278 MachineFunction& MF = CurDAG->getMachineFunction();
1279 MachineMemOperand *MemOp =
1280 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
1281 F: MachineMemOperand::MOLoad, Size: 16, BaseAlignment: Align(8));
1282
1283 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: ResNode), NewMemRefs: {MemOp});
1284 return ResNode;
1285}
1286
1287bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1288 SDValue ElemV = N->getOperand(Num: 2);
1289 auto *ElemN = dyn_cast<ConstantSDNode>(Val&: ElemV);
1290 if (!ElemN)
1291 return false;
1292
1293 unsigned Elem = ElemN->getZExtValue();
1294 EVT VT = N->getValueType(ResNo: 0);
1295 if (Elem >= VT.getVectorNumElements())
1296 return false;
1297
1298 auto *Load = dyn_cast<LoadSDNode>(Val: N->getOperand(Num: 1));
1299 if (!Load || !Load->hasNUsesOfValue(NUses: 1, Value: 0))
1300 return false;
1301 if (Load->getMemoryVT().getSizeInBits() !=
1302 Load->getValueType(ResNo: 0).getSizeInBits())
1303 return false;
1304
1305 SDValue Base, Disp, Index;
1306 if (!selectBDVAddr12Only(Addr: Load->getBasePtr(), Elem: ElemV, Base, Disp, Index) ||
1307 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1308 return false;
1309
1310 SDLoc DL(Load);
1311 SDValue Ops[] = {
1312 N->getOperand(Num: 0), Base, Disp, Index,
1313 CurDAG->getTargetConstant(Val: Elem, DL, VT: MVT::i32), Load->getChain()
1314 };
1315 SDNode *Res = CurDAG->getMachineNode(Opcode, dl: DL, VT1: VT, VT2: MVT::Other, Ops);
1316 ReplaceUses(F: SDValue(Load, 1), T: SDValue(Res, 1));
1317 ReplaceNode(F: N, T: Res);
1318 return true;
1319}
1320
1321bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1322 SDValue Value = Store->getValue();
1323 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1324 return false;
1325 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1326 return false;
1327
1328 SDValue ElemV = Value.getOperand(i: 1);
1329 auto *ElemN = dyn_cast<ConstantSDNode>(Val&: ElemV);
1330 if (!ElemN)
1331 return false;
1332
1333 SDValue Vec = Value.getOperand(i: 0);
1334 EVT VT = Vec.getValueType();
1335 unsigned Elem = ElemN->getZExtValue();
1336 if (Elem >= VT.getVectorNumElements())
1337 return false;
1338
1339 SDValue Base, Disp, Index;
1340 if (!selectBDVAddr12Only(Addr: Store->getBasePtr(), Elem: ElemV, Base, Disp, Index) ||
1341 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1342 return false;
1343
1344 SDLoc DL(Store);
1345 SDValue Ops[] = {
1346 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Val: Elem, DL, VT: MVT::i32),
1347 Store->getChain()
1348 };
1349 ReplaceNode(F: Store, T: CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Other, Ops));
1350 return true;
1351}
1352
1353// Check whether or not the chain ending in StoreNode is suitable for doing
1354// the {load; op; store} to modify transformation.
1355static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1356 SDValue StoredVal, SelectionDAG *CurDAG,
1357 LoadSDNode *&LoadNode,
1358 SDValue &InputChain) {
1359 // Is the stored value result 0 of the operation?
1360 if (StoredVal.getResNo() != 0)
1361 return false;
1362
1363 // Are there other uses of the loaded value than the operation?
1364 if (!StoredVal.getNode()->hasNUsesOfValue(NUses: 1, Value: 0))
1365 return false;
1366
1367 // Is the store non-extending and non-indexed?
1368 if (!ISD::isNormalStore(N: StoreNode) || StoreNode->isNonTemporal())
1369 return false;
1370
1371 SDValue Load = StoredVal->getOperand(Num: 0);
1372 // Is the stored value a non-extending and non-indexed load?
1373 if (!ISD::isNormalLoad(N: Load.getNode()))
1374 return false;
1375
1376 // Return LoadNode by reference.
1377 LoadNode = cast<LoadSDNode>(Val&: Load);
1378
1379 // Is store the only read of the loaded value?
1380 if (!Load.hasOneUse())
1381 return false;
1382
1383 // Is the address of the store the same as the load?
1384 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1385 LoadNode->getOffset() != StoreNode->getOffset())
1386 return false;
1387
1388 // Check if the chain is produced by the load or is a TokenFactor with
1389 // the load output chain as an operand. Return InputChain by reference.
1390 SDValue Chain = StoreNode->getChain();
1391
1392 bool ChainCheck = false;
1393 if (Chain == Load.getValue(R: 1)) {
1394 ChainCheck = true;
1395 InputChain = LoadNode->getChain();
1396 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1397 SmallVector<SDValue, 4> ChainOps;
1398 SmallVector<const SDNode *, 4> LoopWorklist;
1399 SmallPtrSet<const SDNode *, 16> Visited;
1400 const unsigned int Max = 1024;
1401 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1402 SDValue Op = Chain.getOperand(i);
1403 if (Op == Load.getValue(R: 1)) {
1404 ChainCheck = true;
1405 // Drop Load, but keep its chain. No cycle check necessary.
1406 ChainOps.push_back(Elt: Load.getOperand(i: 0));
1407 continue;
1408 }
1409 LoopWorklist.push_back(Elt: Op.getNode());
1410 ChainOps.push_back(Elt: Op);
1411 }
1412
1413 if (ChainCheck) {
1414 // Add the other operand of StoredVal to worklist.
1415 for (SDValue Op : StoredVal->ops())
1416 if (Op.getNode() != LoadNode)
1417 LoopWorklist.push_back(Elt: Op.getNode());
1418
1419 // Check if Load is reachable from any of the nodes in the worklist.
1420 if (SDNode::hasPredecessorHelper(N: Load.getNode(), Visited, Worklist&: LoopWorklist, MaxSteps: Max,
1421 TopologicalPrune: true))
1422 return false;
1423
1424 // Make a new TokenFactor with all the other input chains except
1425 // for the load.
1426 InputChain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain),
1427 VT: MVT::Other, Ops: ChainOps);
1428 }
1429 }
1430 if (!ChainCheck)
1431 return false;
1432
1433 return true;
1434}
1435
1436// Change a chain of {load; op; store} of the same value into a simple op
1437// through memory of that value, if the uses of the modified value and its
1438// address are suitable.
1439//
1440// The tablegen pattern memory operand pattern is currently not able to match
1441// the case where the CC on the original operation are used.
1442//
1443// See the equivalent routine in X86ISelDAGToDAG for further comments.
1444bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1445 StoreSDNode *StoreNode = cast<StoreSDNode>(Val: Node);
1446 SDValue StoredVal = StoreNode->getOperand(Num: 1);
1447 unsigned Opc = StoredVal->getOpcode();
1448 SDLoc DL(StoreNode);
1449
1450 // Before we try to select anything, make sure this is memory operand size
1451 // and opcode we can handle. Note that this must match the code below that
1452 // actually lowers the opcodes.
1453 EVT MemVT = StoreNode->getMemoryVT();
1454 unsigned NewOpc = 0;
1455 bool NegateOperand = false;
1456 switch (Opc) {
1457 default:
1458 return false;
1459 case SystemZISD::SSUBO:
1460 NegateOperand = true;
1461 [[fallthrough]];
1462 case SystemZISD::SADDO:
1463 if (MemVT == MVT::i32)
1464 NewOpc = SystemZ::ASI;
1465 else if (MemVT == MVT::i64)
1466 NewOpc = SystemZ::AGSI;
1467 else
1468 return false;
1469 break;
1470 case SystemZISD::USUBO:
1471 NegateOperand = true;
1472 [[fallthrough]];
1473 case SystemZISD::UADDO:
1474 if (MemVT == MVT::i32)
1475 NewOpc = SystemZ::ALSI;
1476 else if (MemVT == MVT::i64)
1477 NewOpc = SystemZ::ALGSI;
1478 else
1479 return false;
1480 break;
1481 }
1482
1483 LoadSDNode *LoadNode = nullptr;
1484 SDValue InputChain;
1485 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1486 InputChain))
1487 return false;
1488
1489 SDValue Operand = StoredVal.getOperand(i: 1);
1490 auto *OperandC = dyn_cast<ConstantSDNode>(Val&: Operand);
1491 if (!OperandC)
1492 return false;
1493 auto OperandV = OperandC->getAPIntValue();
1494 if (NegateOperand)
1495 OperandV = -OperandV;
1496 if (OperandV.getSignificantBits() > 8)
1497 return false;
1498 Operand = CurDAG->getTargetConstant(Val: OperandV, DL, VT: MemVT);
1499
1500 SDValue Base, Disp;
1501 if (!selectBDAddr20Only(Addr: StoreNode->getBasePtr(), Base, Disp))
1502 return false;
1503
1504 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1505 MachineSDNode *Result =
1506 CurDAG->getMachineNode(Opcode: NewOpc, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
1507 CurDAG->setNodeMemRefs(
1508 N: Result, NewMemRefs: {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1509
1510 ReplaceUses(F: SDValue(StoreNode, 0), T: SDValue(Result, 1));
1511 ReplaceUses(F: SDValue(StoredVal.getNode(), 1), T: SDValue(Result, 0));
1512 CurDAG->RemoveDeadNode(N: Node);
1513 return true;
1514}
1515
1516bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1517 LoadSDNode *Load) const {
1518 // Check that the two memory operands have the same size.
1519 if (Load->getMemoryVT() != Store->getMemoryVT())
1520 return false;
1521
1522 // Volatility stops an access from being decomposed.
1523 if (Load->isVolatile() || Store->isVolatile())
1524 return false;
1525
1526 // There's no chance of overlap if the load is invariant.
1527 if (Load->isInvariant() && Load->isDereferenceable())
1528 return true;
1529
1530 // Otherwise we need to check whether there's an alias.
1531 const Value *V1 = Load->getMemOperand()->getValue();
1532 const Value *V2 = Store->getMemOperand()->getValue();
1533 if (!V1 || !V2)
1534 return false;
1535
1536 // Reject equality.
1537 uint64_t Size = Load->getMemoryVT().getStoreSize();
1538 int64_t End1 = Load->getSrcValueOffset() + Size;
1539 int64_t End2 = Store->getSrcValueOffset() + Size;
1540 if (V1 == V2 && End1 == End2)
1541 return false;
1542
1543 return BatchAA->isNoAlias(LocA: MemoryLocation(V1, End1, Load->getAAInfo()),
1544 LocB: MemoryLocation(V2, End2, Store->getAAInfo()));
1545}
1546
1547bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1548 auto *Store = cast<StoreSDNode>(Val: N);
1549 auto *Load = cast<LoadSDNode>(Val: Store->getValue());
1550
1551 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1552 // instructions.
1553 uint64_t Size = Load->getMemoryVT().getStoreSize();
1554 if (Size > 1 && Size <= 8) {
1555 // Prefer LHRL, LRL and LGRL.
1556 if (SystemZISD::isPCREL(Opcode: Load->getBasePtr().getOpcode()))
1557 return false;
1558 // Prefer STHRL, STRL and STGRL.
1559 if (SystemZISD::isPCREL(Opcode: Store->getBasePtr().getOpcode()))
1560 return false;
1561 }
1562
1563 return canUseBlockOperation(Store, Load);
1564}
1565
1566bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1567 unsigned I) const {
1568 auto *StoreA = cast<StoreSDNode>(Val: N);
1569 auto *LoadA = cast<LoadSDNode>(Val: StoreA->getValue().getOperand(i: 1 - I));
1570 auto *LoadB = cast<LoadSDNode>(Val: StoreA->getValue().getOperand(i: I));
1571 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1572 canUseBlockOperation(Store: StoreA, Load: LoadB);
1573}
1574
1575bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1576
1577 auto *MemAccess = cast<MemSDNode>(Val: N);
1578 auto *LdSt = dyn_cast<LSBaseSDNode>(Val: MemAccess);
1579 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1580 SDValue BasePtr = MemAccess->getBasePtr();
1581 MachineMemOperand *MMO = MemAccess->getMemOperand();
1582 assert(MMO && "Expected a memory operand.");
1583
1584 // The memory access must have a proper alignment and no index register.
1585 // Only load and store nodes have the offset operand (atomic loads do not).
1586 if (MemAccess->getAlign().value() < StoreSize ||
1587 (LdSt && !LdSt->getOffset().isUndef()))
1588 return false;
1589
1590 // The MMO must not have an unaligned offset.
1591 if (MMO->getOffset() % StoreSize != 0)
1592 return false;
1593
1594 // An access to GOT or the Constant Pool is aligned.
1595 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1596 if ((PSV->isGOT() || PSV->isConstantPool()))
1597 return true;
1598
1599 // Check the alignment of a Global Address.
1600 if (BasePtr.getNumOperands())
1601 if (GlobalAddressSDNode *GA =
1602 dyn_cast<GlobalAddressSDNode>(Val: BasePtr.getOperand(i: 0))) {
1603 // The immediate offset must be aligned.
1604 if (GA->getOffset() % StoreSize != 0)
1605 return false;
1606
1607 // The alignment of the symbol itself must be at least the store size.
1608 const GlobalValue *GV = GA->getGlobal();
1609 const DataLayout &DL = GV->getDataLayout();
1610 if (GV->getPointerAlignment(DL).value() < StoreSize)
1611 return false;
1612 }
1613
1614 return true;
1615}
1616
1617ISD::LoadExtType SystemZDAGToDAGISel::getLoadExtType(SDNode *N) const {
1618 ISD::LoadExtType ETy;
1619 if (auto *L = dyn_cast<LoadSDNode>(Val: N))
1620 ETy = L->getExtensionType();
1621 else if (auto *AL = dyn_cast<AtomicSDNode>(Val: N))
1622 ETy = AL->getExtensionType();
1623 else
1624 llvm_unreachable("Unkown load node type.");
1625 return ETy;
1626}
1627
1628void SystemZDAGToDAGISel::Select(SDNode *Node) {
1629 // If we have a custom node, we already have selected!
1630 if (Node->isMachineOpcode()) {
1631 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1632 Node->setNodeId(-1);
1633 return;
1634 }
1635
1636 unsigned Opcode = Node->getOpcode();
1637 switch (Opcode) {
1638 case ISD::OR:
1639 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant)
1640 if (tryRxSBG(N: Node, Opcode: SystemZ::ROSBG))
1641 return;
1642 goto or_xor;
1643
1644 case ISD::XOR:
1645 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant)
1646 if (tryRxSBG(N: Node, Opcode: SystemZ::RXSBG))
1647 return;
1648 // Fall through.
1649 or_xor:
1650 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1651 // split the operation into two. If both operands here happen to be
1652 // constant, leave this to common code to optimize.
1653 if (Node->getValueType(ResNo: 0) == MVT::i64 &&
1654 Node->getOperand(Num: 0).getOpcode() != ISD::Constant)
1655 if (auto *Op1 = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
1656 uint64_t Val = Op1->getZExtValue();
1657 // Don't split the operation if we can match one of the combined
1658 // logical operations provided by miscellaneous-extensions-3.
1659 if (Subtarget->hasMiscellaneousExtensions3()) {
1660 unsigned ChildOpcode = Node->getOperand(Num: 0).getOpcode();
1661 // Check whether this expression matches NAND/NOR/NXOR.
1662 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1663 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1664 ChildOpcode == ISD::XOR)
1665 break;
1666 // Check whether this expression matches OR-with-complement
1667 // (or matches an alternate pattern for NXOR).
1668 if (ChildOpcode == ISD::XOR) {
1669 auto Op0 = Node->getOperand(Num: 0);
1670 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Val: Op0->getOperand(Num: 1)))
1671 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1672 break;
1673 }
1674 }
1675 // Don't split an XOR with -1 as LCGR/AGHI is more compact.
1676 if (Opcode == ISD::XOR && Op1->isAllOnes())
1677 break;
1678 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1679 splitLargeImmediate(Opcode, Node, Op0: Node->getOperand(Num: 0),
1680 UpperVal: Val - uint32_t(Val), LowerVal: uint32_t(Val));
1681 return;
1682 }
1683 }
1684 break;
1685
1686 case ISD::AND:
1687 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant) {
1688 if (tryRxSBG(N: Node, Opcode: SystemZ::RNSBG))
1689 return;
1690 } else {
1691 // Use patterns for zero-extending of vector element extraction.
1692 if (Node->getValueType(ResNo: 0) == MVT::i64 &&
1693 Node->getOperand(Num: 0)->getOpcode() == ISD::ANY_EXTEND) {
1694 SDValue Input = Node->getOperand(Num: 0)->getOperand(Num: 0);
1695 uint64_t Mask =
1696 cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))->getZExtValue();
1697 if (Input->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1698 EVT VecVT = Input->getOperand(Num: 0)->getValueType(ResNo: 0);
1699 unsigned EltBits = VecVT.getScalarSizeInBits();
1700 if (allOnes(Count: EltBits) == Mask)
1701 break;
1702 }
1703 }
1704 }
1705
1706 [[fallthrough]];
1707 case ISD::ROTL:
1708 case ISD::SHL:
1709 case ISD::SRL:
1710 case ISD::ZERO_EXTEND:
1711 if (tryRISBGZero(N: Node))
1712 return;
1713 break;
1714
1715 case ISD::BSWAP:
1716 if (Node->getValueType(ResNo: 0) == MVT::i128) {
1717 SDLoc DL(Node);
1718 SDValue Src = Node->getOperand(Num: 0);
1719 Src = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Src);
1720
1721 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1722 SDNode *Mask = loadPoolVectorConstant(Val: APInt(128, Bytes), VT: MVT::v16i8, DL);
1723 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) };
1724 SDValue Res = SDValue(CurDAG->getMachineNode(Opcode: SystemZ::VPERM, dl: DL,
1725 VT: MVT::v16i8, Ops), 0);
1726
1727 Res = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i128, Operand: Res);
1728 SDNode *ResNode = Res.getNode();
1729 ReplaceNode(F: Node, T: ResNode);
1730 SelectCode(N: Src.getNode());
1731 SelectCode(N: ResNode);
1732 return;
1733 }
1734 break;
1735
1736 case ISD::Constant:
1737 // If this is a 64-bit constant that is out of the range of LLILF,
1738 // LLIHF and LGFI, split it into two 32-bit pieces.
1739 if (Node->getValueType(ResNo: 0) == MVT::i64) {
1740 uint64_t Val = Node->getAsZExtVal();
1741 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(x: Val)) {
1742 splitLargeImmediate(Opcode: ISD::OR, Node, Op0: SDValue(), UpperVal: Val - uint32_t(Val),
1743 LowerVal: uint32_t(Val));
1744 return;
1745 }
1746 }
1747 if (Node->getValueType(ResNo: 0) == MVT::i128) {
1748 const APInt &Val = Node->getAsAPIntVal();
1749 SystemZVectorConstantInfo VCI(Val);
1750 if (VCI.isVectorConstantLegal(Subtarget: *Subtarget)) {
1751 loadVectorConstant(VCI, Node);
1752 return;
1753 }
1754 // If we can't materialize the constant we need to use a literal pool.
1755 SDNode *ResNode = loadPoolVectorConstant(Val, VT: MVT::i128, DL: SDLoc(Node));
1756 ReplaceNode(F: Node, T: ResNode);
1757 return;
1758 }
1759 break;
1760
1761 case SystemZISD::SELECT_CCMASK: {
1762 SDValue Op0 = Node->getOperand(Num: 0);
1763 SDValue Op1 = Node->getOperand(Num: 1);
1764 // Prefer to put any load first, so that it can be matched as a
1765 // conditional load. Likewise for constants in range for LOCHI.
1766 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1767 (Subtarget->hasLoadStoreOnCond2() &&
1768 Node->getValueType(ResNo: 0).isInteger() &&
1769 Node->getValueType(ResNo: 0).getSizeInBits() <= 64 &&
1770 Op1.getOpcode() == ISD::Constant &&
1771 isInt<16>(x: cast<ConstantSDNode>(Val&: Op1)->getSExtValue()) &&
1772 !(Op0.getOpcode() == ISD::Constant &&
1773 isInt<16>(x: cast<ConstantSDNode>(Val&: Op0)->getSExtValue())))) {
1774 SDValue CCValid = Node->getOperand(Num: 2);
1775 SDValue CCMask = Node->getOperand(Num: 3);
1776 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal();
1777 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal();
1778 // Invert the condition.
1779 CCMask = CurDAG->getTargetConstant(Val: ConstCCValid ^ ConstCCMask,
1780 DL: SDLoc(Node), VT: CCMask.getValueType());
1781 SDValue Op4 = Node->getOperand(Num: 4);
1782 SDNode *UpdatedNode =
1783 CurDAG->UpdateNodeOperands(N: Node, Op1, Op2: Op0, Op3: CCValid, Op4: CCMask, Op5: Op4);
1784 if (UpdatedNode != Node) {
1785 // In case this node already exists then replace Node with it.
1786 ReplaceNode(F: Node, T: UpdatedNode);
1787 Node = UpdatedNode;
1788 }
1789 }
1790 break;
1791 }
1792
1793 case ISD::INSERT_VECTOR_ELT: {
1794 EVT VT = Node->getValueType(ResNo: 0);
1795 unsigned ElemBitSize = VT.getScalarSizeInBits();
1796 if (ElemBitSize == 32) {
1797 if (tryGather(N: Node, Opcode: SystemZ::VGEF))
1798 return;
1799 } else if (ElemBitSize == 64) {
1800 if (tryGather(N: Node, Opcode: SystemZ::VGEG))
1801 return;
1802 }
1803 break;
1804 }
1805
1806 case ISD::BUILD_VECTOR: {
1807 auto *BVN = cast<BuildVectorSDNode>(Val: Node);
1808 SystemZVectorConstantInfo VCI(BVN);
1809 if (VCI.isVectorConstantLegal(Subtarget: *Subtarget)) {
1810 loadVectorConstant(VCI, Node);
1811 return;
1812 }
1813 break;
1814 }
1815
1816 case ISD::ConstantFP: {
1817 APFloat Imm = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1818 if (Imm.isZero() || Imm.isNegZero())
1819 break;
1820 SystemZVectorConstantInfo VCI(Imm);
1821 bool Success = VCI.isVectorConstantLegal(Subtarget: *Subtarget); (void)Success;
1822 assert(Success && "Expected legal FP immediate");
1823 loadVectorConstant(VCI, Node);
1824 return;
1825 }
1826
1827 case ISD::STORE: {
1828 if (tryFoldLoadStoreIntoMemOperand(Node))
1829 return;
1830 auto *Store = cast<StoreSDNode>(Val: Node);
1831 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1832 if (ElemBitSize == 32) {
1833 if (tryScatter(Store, Opcode: SystemZ::VSCEF))
1834 return;
1835 } else if (ElemBitSize == 64) {
1836 if (tryScatter(Store, Opcode: SystemZ::VSCEG))
1837 return;
1838 }
1839 break;
1840 }
1841
1842 case ISD::ATOMIC_STORE: {
1843 auto *AtomOp = cast<AtomicSDNode>(Val: Node);
1844 // Replace the atomic_store with a regular store and select it. This is
1845 // ok since we know all store instructions <= 8 bytes are atomic, and the
1846 // 16 byte case is already handled during lowering.
1847 StoreSDNode *St = cast<StoreSDNode>(Val: CurDAG->getTruncStore(
1848 Chain: AtomOp->getChain(), dl: SDLoc(AtomOp), Val: AtomOp->getVal(),
1849 Ptr: AtomOp->getBasePtr(), SVT: AtomOp->getMemoryVT(), MMO: AtomOp->getMemOperand()));
1850 assert(St->getMemOperand()->isAtomic() && "Broken MMO.");
1851 SDNode *Chain = St;
1852 // We have to enforce sequential consistency by performing a
1853 // serialization operation after the store.
1854 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1855 Chain = CurDAG->getMachineNode(Opcode: SystemZ::Serialize, dl: SDLoc(AtomOp),
1856 VT: MVT::Other, Op1: SDValue(Chain, 0));
1857 ReplaceNode(F: Node, T: Chain);
1858 SelectCode(N: St);
1859 return;
1860 }
1861 }
1862
1863 SelectCode(N: Node);
1864}
1865
1866bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1867 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1868 std::vector<SDValue> &OutOps) {
1869 SystemZAddressingMode::AddrForm Form;
1870 SystemZAddressingMode::DispRange DispRange;
1871 SDValue Base, Disp, Index;
1872
1873 switch(ConstraintID) {
1874 default:
1875 llvm_unreachable("Unexpected asm memory constraint");
1876 case InlineAsm::ConstraintCode::i:
1877 case InlineAsm::ConstraintCode::Q:
1878 case InlineAsm::ConstraintCode::ZQ:
1879 // Accept an address with a short displacement, but no index.
1880 Form = SystemZAddressingMode::FormBD;
1881 DispRange = SystemZAddressingMode::Disp12Only;
1882 break;
1883 case InlineAsm::ConstraintCode::R:
1884 case InlineAsm::ConstraintCode::ZR:
1885 // Accept an address with a short displacement and an index.
1886 Form = SystemZAddressingMode::FormBDXNormal;
1887 DispRange = SystemZAddressingMode::Disp12Only;
1888 break;
1889 case InlineAsm::ConstraintCode::S:
1890 case InlineAsm::ConstraintCode::ZS:
1891 // Accept an address with a long displacement, but no index.
1892 Form = SystemZAddressingMode::FormBD;
1893 DispRange = SystemZAddressingMode::Disp20Only;
1894 break;
1895 case InlineAsm::ConstraintCode::T:
1896 case InlineAsm::ConstraintCode::m:
1897 case InlineAsm::ConstraintCode::o:
1898 case InlineAsm::ConstraintCode::p:
1899 case InlineAsm::ConstraintCode::ZT:
1900 // Accept an address with a long displacement and an index.
1901 // m works the same as T, as this is the most general case.
1902 // We don't really have any special handling of "offsettable"
1903 // memory addresses, so just treat o the same as m.
1904 Form = SystemZAddressingMode::FormBDXNormal;
1905 DispRange = SystemZAddressingMode::Disp20Only;
1906 break;
1907 }
1908
1909 if (selectBDXAddr(Form, DR: DispRange, Addr: Op, Base, Disp, Index)) {
1910 const TargetRegisterClass *TRC =
1911 Subtarget->getRegisterInfo()->getPointerRegClass();
1912 SDLoc DL(Base);
1913 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL, VT: MVT::i32);
1914
1915 // Make sure that the base address doesn't go into %r0.
1916 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1917 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1918 Base.getOpcode() != ISD::Register) {
1919 Base =
1920 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
1921 dl: DL, VT: Base.getValueType(),
1922 Op1: Base, Op2: RC), 0);
1923 }
1924
1925 // Make sure that the index register isn't assigned to %r0 either.
1926 if (Index.getOpcode() != ISD::Register) {
1927 Index =
1928 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
1929 dl: DL, VT: Index.getValueType(),
1930 Op1: Index, Op2: RC), 0);
1931 }
1932
1933 OutOps.push_back(x: Base);
1934 OutOps.push_back(x: Disp);
1935 OutOps.push_back(x: Index);
1936 return false;
1937 }
1938
1939 return true;
1940}
1941
1942// IsProfitableToFold - Returns true if is profitable to fold the specific
1943// operand node N of U during instruction selection that starts at Root.
1944bool
1945SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1946 SDNode *Root) const {
1947 // We want to avoid folding a LOAD into an ICMP node if as a result
1948 // we would be forced to spill the condition code into a GPR.
1949 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1950 if (!N.hasOneUse() || !U->hasOneUse())
1951 return false;
1952
1953 // The user of the CC value will usually be a CopyToReg into the
1954 // physical CC register, which in turn is glued and chained to the
1955 // actual instruction that uses the CC value. Bail out if we have
1956 // anything else than that.
1957 SDNode *CCUser = *U->user_begin();
1958 SDNode *CCRegUser = nullptr;
1959 if (CCUser->getOpcode() == ISD::CopyToReg ||
1960 cast<RegisterSDNode>(Val: CCUser->getOperand(Num: 1))->getReg() == SystemZ::CC) {
1961 for (auto *U : CCUser->users()) {
1962 if (CCRegUser == nullptr)
1963 CCRegUser = U;
1964 else if (CCRegUser != U)
1965 return false;
1966 }
1967 }
1968 if (CCRegUser == nullptr)
1969 return false;
1970
1971 // If the actual instruction is a branch, the only thing that remains to be
1972 // checked is whether the CCUser chain is a predecessor of the load.
1973 if (CCRegUser->isMachineOpcode() &&
1974 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1975 return !N->isPredecessorOf(N: CCUser->getOperand(Num: 0).getNode());
1976
1977 // Otherwise, the instruction may have multiple operands, and we need to
1978 // verify that none of them are a predecessor of the load. This is exactly
1979 // the same check that would be done by common code if the CC setter were
1980 // glued to the CC user, so simply invoke that check here.
1981 if (!IsLegalToFold(N, U, Root: CCRegUser, OptLevel, IgnoreChains: false))
1982 return false;
1983 }
1984
1985 return true;
1986}
1987
1988namespace {
1989// Represents a sequence for extracting a 0/1 value from an IPM result:
1990// (((X ^ XORValue) + AddValue) >> Bit)
1991struct IPMConversion {
1992 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1993 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1994
1995 int64_t XORValue;
1996 int64_t AddValue;
1997 unsigned Bit;
1998};
1999} // end anonymous namespace
2000
2001// Return a sequence for getting a 1 from an IPM result when CC has a
2002// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
2003// The handling of CC values outside CCValid doesn't matter.
2004static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
2005 // Deal with cases where the result can be taken directly from a bit
2006 // of the IPM result.
2007 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
2008 return IPMConversion(0, 0, SystemZ::IPM_CC);
2009 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
2010 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
2011
2012 // Deal with cases where we can add a value to force the sign bit
2013 // to contain the right value. Putting the bit in 31 means we can
2014 // use SRL rather than RISBG(L), and also makes it easier to get a
2015 // 0/-1 value, so it has priority over the other tests below.
2016 //
2017 // These sequences rely on the fact that the upper two bits of the
2018 // IPM result are zero.
2019 uint64_t TopBit = uint64_t(1) << 31;
2020 if (CCMask == (CCValid & SystemZ::CCMASK_0))
2021 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
2022 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
2023 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
2024 if (CCMask == (CCValid & (SystemZ::CCMASK_0
2025 | SystemZ::CCMASK_1
2026 | SystemZ::CCMASK_2)))
2027 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
2028 if (CCMask == (CCValid & SystemZ::CCMASK_3))
2029 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
2030 if (CCMask == (CCValid & (SystemZ::CCMASK_1
2031 | SystemZ::CCMASK_2
2032 | SystemZ::CCMASK_3)))
2033 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
2034
2035 // Next try inverting the value and testing a bit. 0/1 could be
2036 // handled this way too, but we dealt with that case above.
2037 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
2038 return IPMConversion(-1, 0, SystemZ::IPM_CC);
2039
2040 // Handle cases where adding a value forces a non-sign bit to contain
2041 // the right value.
2042 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
2043 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
2044 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
2045 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
2046
2047 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
2048 // can be done by inverting the low CC bit and applying one of the
2049 // sign-based extractions above.
2050 if (CCMask == (CCValid & SystemZ::CCMASK_1))
2051 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
2052 if (CCMask == (CCValid & SystemZ::CCMASK_2))
2053 return IPMConversion(1 << SystemZ::IPM_CC,
2054 TopBit - (3 << SystemZ::IPM_CC), 31);
2055 if (CCMask == (CCValid & (SystemZ::CCMASK_0
2056 | SystemZ::CCMASK_1
2057 | SystemZ::CCMASK_3)))
2058 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
2059 if (CCMask == (CCValid & (SystemZ::CCMASK_0
2060 | SystemZ::CCMASK_2
2061 | SystemZ::CCMASK_3)))
2062 return IPMConversion(1 << SystemZ::IPM_CC,
2063 TopBit - (1 << SystemZ::IPM_CC), 31);
2064
2065 llvm_unreachable("Unexpected CC combination");
2066}
2067
2068SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
2069 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0));
2070 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
2071 if (!TrueOp || !FalseOp)
2072 return SDValue();
2073 if (FalseOp->getZExtValue() != 0)
2074 return SDValue();
2075 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
2076 return SDValue();
2077
2078 auto *CCValidOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
2079 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 3));
2080 if (!CCValidOp || !CCMaskOp)
2081 return SDValue();
2082 int CCValid = CCValidOp->getZExtValue();
2083 int CCMask = CCMaskOp->getZExtValue();
2084
2085 SDLoc DL(Node);
2086 SDValue CCReg = Node->getOperand(Num: 4);
2087 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
2088 SDValue Result = CurDAG->getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
2089
2090 if (IPM.XORValue)
2091 Result = CurDAG->getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Result,
2092 N2: CurDAG->getConstant(Val: IPM.XORValue, DL, VT: MVT::i32));
2093
2094 if (IPM.AddValue)
2095 Result =
2096 CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: Result,
2097 N2: CurDAG->getSignedConstant(Val: IPM.AddValue, DL, VT: MVT::i32));
2098
2099 EVT VT = Node->getValueType(ResNo: 0);
2100 if (VT == MVT::i32 && IPM.Bit == 31) {
2101 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
2102 Result = CurDAG->getNode(Opcode: ShiftOp, DL, VT: MVT::i32, N1: Result,
2103 N2: CurDAG->getConstant(Val: IPM.Bit, DL, VT: MVT::i32));
2104 } else {
2105 if (VT != MVT::i32)
2106 Result = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Result);
2107
2108 if (TrueOp->getSExtValue() == 1) {
2109 // The SHR/AND sequence should get optimized to an RISBG.
2110 Result = CurDAG->getNode(Opcode: ISD::SRL, DL, VT, N1: Result,
2111 N2: CurDAG->getConstant(Val: IPM.Bit, DL, VT: MVT::i32));
2112 Result = CurDAG->getNode(Opcode: ISD::AND, DL, VT, N1: Result,
2113 N2: CurDAG->getConstant(Val: 1, DL, VT));
2114 } else {
2115 // Sign-extend from IPM.Bit using a pair of shifts.
2116 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
2117 int SraAmt = VT.getSizeInBits() - 1;
2118 Result = CurDAG->getNode(Opcode: ISD::SHL, DL, VT, N1: Result,
2119 N2: CurDAG->getConstant(Val: ShlAmt, DL, VT: MVT::i32));
2120 Result = CurDAG->getNode(Opcode: ISD::SRA, DL, VT, N1: Result,
2121 N2: CurDAG->getConstant(Val: SraAmt, DL, VT: MVT::i32));
2122 }
2123 }
2124
2125 return Result;
2126}
2127
2128bool SystemZDAGToDAGISel::shouldSelectForReassoc(SDNode *N) const {
2129 EVT VT = N->getValueType(ResNo: 0);
2130 assert(VT.isFloatingPoint() && "Expected FP SDNode");
2131 return N->getFlags().hasAllowReassociation() &&
2132 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&
2133 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&
2134 !N->isStrictFPOpcode();
2135}
2136
2137void SystemZDAGToDAGISel::PreprocessISelDAG() {
2138 // If we have conditional immediate loads, we always prefer
2139 // using those over an IPM sequence.
2140 if (Subtarget->hasLoadStoreOnCond2())
2141 return;
2142
2143 bool MadeChange = false;
2144
2145 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
2146 E = CurDAG->allnodes_end();
2147 I != E;) {
2148 SDNode *N = &*I++;
2149 if (N->use_empty())
2150 continue;
2151
2152 SDValue Res;
2153 switch (N->getOpcode()) {
2154 default: break;
2155 case SystemZISD::SELECT_CCMASK:
2156 Res = expandSelectBoolean(Node: N);
2157 break;
2158 }
2159
2160 if (Res) {
2161 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
2162 LLVM_DEBUG(N->dump(CurDAG));
2163 LLVM_DEBUG(dbgs() << "\nNew: ");
2164 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
2165 LLVM_DEBUG(dbgs() << "\n");
2166
2167 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
2168 MadeChange = true;
2169 }
2170 }
2171
2172 if (MadeChange)
2173 CurDAG->RemoveDeadNodes();
2174}
2175