1//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the SystemZ target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZTargetMachine.h"
15#include "llvm/Analysis/AliasAnalysis.h"
16#include "llvm/CodeGen/SelectionDAGISel.h"
17#include "llvm/IR/Module.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/KnownBits.h"
20#include "llvm/Support/raw_ostream.h"
21
22using namespace llvm;
23
24#define DEBUG_TYPE "systemz-isel"
25#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
26
27namespace {
28// Used to build addressing modes.
29struct SystemZAddressingMode {
30 // The shape of the address.
31 enum AddrForm {
32 // base+displacement
33 FormBD,
34
35 // base+displacement+index for load and store operands
36 FormBDXNormal,
37
38 // base+displacement+index for load address operands
39 FormBDXLA,
40
41 // base+displacement+index+ADJDYNALLOC
42 FormBDXDynAlloc
43 };
44 AddrForm Form;
45
46 // The type of displacement. The enum names here correspond directly
47 // to the definitions in SystemZOperand.td. We could split them into
48 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
49 enum DispRange {
50 Disp12Only,
51 Disp12Pair,
52 Disp20Only,
53 Disp20Only128,
54 Disp20Pair
55 };
56 DispRange DR;
57
58 // The parts of the address. The address is equivalent to:
59 //
60 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
61 SDValue Base;
62 int64_t Disp;
63 SDValue Index;
64 bool IncludesDynAlloc;
65
66 SystemZAddressingMode(AddrForm form, DispRange dr)
67 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
68
69 // True if the address can have an index register.
70 bool hasIndexField() { return Form != FormBD; }
71
72 // True if the address can (and must) include ADJDYNALLOC.
73 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
74
75 void dump(const llvm::SelectionDAG *DAG) {
76 errs() << "SystemZAddressingMode " << this << '\n';
77
78 errs() << " Base ";
79 if (Base.getNode())
80 Base.getNode()->dump(G: DAG);
81 else
82 errs() << "null\n";
83
84 if (hasIndexField()) {
85 errs() << " Index ";
86 if (Index.getNode())
87 Index.getNode()->dump(G: DAG);
88 else
89 errs() << "null\n";
90 }
91
92 errs() << " Disp " << Disp;
93 if (IncludesDynAlloc)
94 errs() << " + ADJDYNALLOC";
95 errs() << '\n';
96 }
97};
98
99// Return a mask with Count low bits set.
100static uint64_t allOnes(unsigned int Count) {
101 assert(Count <= 64);
102 if (Count > 63)
103 return UINT64_MAX;
104 return (uint64_t(1) << Count) - 1;
105}
106
107// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
108// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
109// Rotate (I5). The combined operand value is effectively:
110//
111// (or (rotl Input, Rotate), ~Mask)
112//
113// for RNSBG and:
114//
115// (and (rotl Input, Rotate), Mask)
116//
117// otherwise. The output value has BitSize bits, although Input may be
118// narrower (in which case the upper bits are don't care), or wider (in which
119// case the result will be truncated as part of the operation).
120struct RxSBGOperands {
121 RxSBGOperands(unsigned Op, SDValue N)
122 : Opcode(Op), BitSize(N.getValueSizeInBits()),
123 Mask(allOnes(Count: BitSize)), Input(N), Start(64 - BitSize), End(63),
124 Rotate(0) {}
125
126 unsigned Opcode;
127 unsigned BitSize;
128 uint64_t Mask;
129 SDValue Input;
130 unsigned Start;
131 unsigned End;
132 unsigned Rotate;
133};
134
135class SystemZDAGToDAGISel : public SelectionDAGISel {
136 const SystemZSubtarget *Subtarget;
137
138 // Used by SystemZOperands.td to create integer constants.
139 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
140 return CurDAG->getTargetConstant(Val: Imm, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
141 }
142
143 const SystemZTargetMachine &getTargetMachine() const {
144 return static_cast<const SystemZTargetMachine &>(TM);
145 }
146
147 const SystemZInstrInfo *getInstrInfo() const {
148 return Subtarget->getInstrInfo();
149 }
150
151 // Try to fold more of the base or index of AM into AM, where IsBase
152 // selects between the base and index.
153 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
154
155 // Try to describe N in AM, returning true on success.
156 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
157
158 // Extract individual target operands from matched address AM.
159 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
160 SDValue &Base, SDValue &Disp) const;
161 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
162 SDValue &Base, SDValue &Disp, SDValue &Index) const;
163
164 // Try to match Addr as a FormBD address with displacement type DR.
165 // Return true on success, storing the base and displacement in
166 // Base and Disp respectively.
167 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
168 SDValue &Base, SDValue &Disp) const;
169
170 // Try to match Addr as a FormBDX address with displacement type DR.
171 // Return true on success and if the result had no index. Store the
172 // base and displacement in Base and Disp respectively.
173 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
174 SDValue &Base, SDValue &Disp) const;
175
176 // Try to match Addr as a FormBDX* address of form Form with
177 // displacement type DR. Return true on success, storing the base,
178 // displacement and index in Base, Disp and Index respectively.
179 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
180 SystemZAddressingMode::DispRange DR, SDValue Addr,
181 SDValue &Base, SDValue &Disp, SDValue &Index) const;
182
183 // PC-relative address matching routines used by SystemZOperands.td.
184 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
185 if (SystemZISD::isPCREL(Opcode: Addr.getOpcode())) {
186 Target = Addr.getOperand(i: 0);
187 return true;
188 }
189 return false;
190 }
191
192 // BD matching routines used by SystemZOperands.td.
193 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
194 return selectBDAddr(DR: SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
195 }
196 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
197 return selectBDAddr(DR: SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
198 }
199 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
200 return selectBDAddr(DR: SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
201 }
202 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
203 return selectBDAddr(DR: SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
204 }
205
206 // MVI matching routines used by SystemZOperands.td.
207 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
208 return selectMVIAddr(DR: SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
209 }
210 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
211 return selectMVIAddr(DR: SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
212 }
213
214 // BDX matching routines used by SystemZOperands.td.
215 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
216 SDValue &Index) const {
217 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
218 DR: SystemZAddressingMode::Disp12Only,
219 Addr, Base, Disp, Index);
220 }
221 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
222 SDValue &Index) const {
223 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
224 DR: SystemZAddressingMode::Disp12Pair,
225 Addr, Base, Disp, Index);
226 }
227 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
228 SDValue &Index) const {
229 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXDynAlloc,
230 DR: SystemZAddressingMode::Disp12Only,
231 Addr, Base, Disp, Index);
232 }
233 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
234 SDValue &Index) const {
235 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
236 DR: SystemZAddressingMode::Disp20Only,
237 Addr, Base, Disp, Index);
238 }
239 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
240 SDValue &Index) const {
241 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
242 DR: SystemZAddressingMode::Disp20Only128,
243 Addr, Base, Disp, Index);
244 }
245 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
246 SDValue &Index) const {
247 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXNormal,
248 DR: SystemZAddressingMode::Disp20Pair,
249 Addr, Base, Disp, Index);
250 }
251 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
252 SDValue &Index) const {
253 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXLA,
254 DR: SystemZAddressingMode::Disp12Pair,
255 Addr, Base, Disp, Index);
256 }
257 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
258 SDValue &Index) const {
259 return selectBDXAddr(Form: SystemZAddressingMode::FormBDXLA,
260 DR: SystemZAddressingMode::Disp20Pair,
261 Addr, Base, Disp, Index);
262 }
263
264 // Try to match Addr as an address with a base, 12-bit displacement
265 // and index, where the index is element Elem of a vector.
266 // Return true on success, storing the base, displacement and vector
267 // in Base, Disp and Index respectively.
268 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
269 SDValue &Disp, SDValue &Index) const;
270
271 // Check whether (or Op (and X InsertMask)) is effectively an insertion
272 // of X into bits InsertMask of some Y != Op. Return true if so and
273 // set Op to that Y.
274 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
275
276 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
277 // Return true on success.
278 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
279
280 // Try to fold some of RxSBG.Input into other fields of RxSBG.
281 // Return true on success.
282 bool expandRxSBG(RxSBGOperands &RxSBG) const;
283
284 // Return an undefined value of type VT.
285 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
286
287 // Convert N to VT, if it isn't already.
288 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
289
290 // Try to implement AND or shift node N using RISBG with the zero flag set.
291 // Return the selected node on success, otherwise return null.
292 bool tryRISBGZero(SDNode *N);
293
294 // Try to use RISBG or Opcode to implement OR or XOR node N.
295 // Return the selected node on success, otherwise return null.
296 bool tryRxSBG(SDNode *N, unsigned Opcode);
297
298 // If Op0 is null, then Node is a constant that can be loaded using:
299 //
300 // (Opcode UpperVal LowerVal)
301 //
302 // If Op0 is nonnull, then Node can be implemented using:
303 //
304 // (Opcode (Opcode Op0 UpperVal) LowerVal)
305 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
306 uint64_t UpperVal, uint64_t LowerVal);
307
308 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
309 SDNode *Node);
310
311 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL);
312
313 // Try to use gather instruction Opcode to implement vector insertion N.
314 bool tryGather(SDNode *N, unsigned Opcode);
315
316 // Try to use scatter instruction Opcode to implement store Store.
317 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
318
319 // Change a chain of {load; op; store} of the same value into a simple op
320 // through memory of that value, if the uses of the modified value and its
321 // address are suitable.
322 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
323
324 // Return true if Load and Store are loads and stores of the same size
325 // and are guaranteed not to overlap. Such operations can be implemented
326 // using block (SS-format) instructions.
327 //
328 // Partial overlap would lead to incorrect code, since the block operations
329 // are logically bytewise, even though they have a fast path for the
330 // non-overlapping case. We also need to avoid full overlap (i.e. two
331 // addresses that might be equal at run time) because although that case
332 // would be handled correctly, it might be implemented by millicode.
333 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
334
335 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
336 // from Y to X.
337 bool storeLoadCanUseMVC(SDNode *N) const;
338
339 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
340 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
341 // to X.
342 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
343
344 // Return true if N (a load or a store) fullfills the alignment
345 // requirements for a PC-relative access.
346 bool storeLoadIsAligned(SDNode *N) const;
347
348 // Return the load extension type of a load or atomic load.
349 ISD::LoadExtType getLoadExtType(SDNode *N) const;
350
351 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
352 SDValue expandSelectBoolean(SDNode *Node);
353
354 // Return true if the flags of N and the subtarget allows for
355 // reassociation, in which case a reg/reg opcode is needed as input to the
356 // MachineCombiner.
357 bool shouldSelectForReassoc(SDNode *N) const;
358
359public:
360 SystemZDAGToDAGISel() = delete;
361
362 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
363 : SelectionDAGISel(TM, OptLevel) {}
364
365 bool runOnMachineFunction(MachineFunction &MF) override {
366 const Function &F = MF.getFunction();
367 if (F.getFnAttribute(Kind: "fentry-call").getValueAsString() != "true") {
368 if (F.hasFnAttribute(Kind: "mnop-mcount"))
369 report_fatal_error(reason: "mnop-mcount only supported with fentry-call");
370 if (F.hasFnAttribute(Kind: "mrecord-mcount"))
371 report_fatal_error(reason: "mrecord-mcount only supported with fentry-call");
372 }
373 if (F.getParent()->getStackProtectorGuard() != "global") {
374 if (F.getParent()->hasStackProtectorGuardRecord())
375 report_fatal_error(reason: "mstack-protector-guard-record only supported with "
376 "mstack-protector-guard=global");
377 }
378 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
379 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
380 }
381
382 // Override SelectionDAGISel.
383 void Select(SDNode *Node) override;
384 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
385 InlineAsm::ConstraintCode ConstraintID,
386 std::vector<SDValue> &OutOps) override;
387 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
388 void PreprocessISelDAG() override;
389
390 // Include the pieces autogenerated from the target description.
391 #include "SystemZGenDAGISel.inc"
392};
393
394class SystemZDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
395public:
396 static char ID;
397 explicit SystemZDAGToDAGISelLegacy(SystemZTargetMachine &TM,
398 CodeGenOptLevel OptLevel)
399 : SelectionDAGISelLegacy(
400 ID, std::make_unique<SystemZDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
401};
402} // end anonymous namespace
403
404char SystemZDAGToDAGISelLegacy::ID = 0;
405
406INITIALIZE_PASS(SystemZDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
407
408FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
409 CodeGenOptLevel OptLevel) {
410 return new SystemZDAGToDAGISelLegacy(TM, OptLevel);
411}
412
413// Return true if Val should be selected as a displacement for an address
414// with range DR. Here we're interested in the range of both the instruction
415// described by DR and of any pairing instruction.
416static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
417 switch (DR) {
418 case SystemZAddressingMode::Disp12Only:
419 return isUInt<12>(x: Val);
420
421 case SystemZAddressingMode::Disp12Pair:
422 case SystemZAddressingMode::Disp20Only:
423 case SystemZAddressingMode::Disp20Pair:
424 return isInt<20>(x: Val);
425
426 case SystemZAddressingMode::Disp20Only128:
427 return isInt<20>(x: Val) && isInt<20>(x: Val + 8);
428 }
429 llvm_unreachable("Unhandled displacement range");
430}
431
432// Change the base or index in AM to Value, where IsBase selects
433// between the base and index.
434static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
435 SDValue Value) {
436 if (IsBase)
437 AM.Base = Value;
438 else
439 AM.Index = Value;
440}
441
442// The base or index of AM is equivalent to Value + ADJDYNALLOC,
443// where IsBase selects between the base and index. Try to fold the
444// ADJDYNALLOC into AM.
445static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
446 SDValue Value) {
447 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
448 changeComponent(AM, IsBase, Value);
449 AM.IncludesDynAlloc = true;
450 return true;
451 }
452 return false;
453}
454
455// The base of AM is equivalent to Base + Index. Try to use Index as
456// the index register.
457static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
458 SDValue Index) {
459 if (AM.hasIndexField() && !AM.Index.getNode()) {
460 AM.Base = Base;
461 AM.Index = Index;
462 return true;
463 }
464 return false;
465}
466
467// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
468// between the base and index. Try to fold Op1 into AM's displacement.
469static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
470 SDValue Op0, uint64_t Op1) {
471 // First try adjusting the displacement.
472 int64_t TestDisp = AM.Disp + Op1;
473 if (selectDisp(DR: AM.DR, Val: TestDisp)) {
474 changeComponent(AM, IsBase, Value: Op0);
475 AM.Disp = TestDisp;
476 return true;
477 }
478
479 // We could consider forcing the displacement into a register and
480 // using it as an index, but it would need to be carefully tuned.
481 return false;
482}
483
484bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
485 bool IsBase) const {
486 SDValue N = IsBase ? AM.Base : AM.Index;
487 unsigned Opcode = N.getOpcode();
488 // Look through no-op truncations.
489 if (Opcode == ISD::TRUNCATE && N.getOperand(i: 0).getValueSizeInBits() <= 64) {
490 N = N.getOperand(i: 0);
491 Opcode = N.getOpcode();
492 }
493 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(Op: N)) {
494 SDValue Op0 = N.getOperand(i: 0);
495 SDValue Op1 = N.getOperand(i: 1);
496
497 unsigned Op0Code = Op0->getOpcode();
498 unsigned Op1Code = Op1->getOpcode();
499
500 if (Op0Code == SystemZISD::ADJDYNALLOC)
501 return expandAdjDynAlloc(AM, IsBase, Value: Op1);
502 if (Op1Code == SystemZISD::ADJDYNALLOC)
503 return expandAdjDynAlloc(AM, IsBase, Value: Op0);
504
505 if (Op0Code == ISD::Constant)
506 return expandDisp(AM, IsBase, Op0: Op1,
507 Op1: cast<ConstantSDNode>(Val&: Op0)->getSExtValue());
508 if (Op1Code == ISD::Constant)
509 return expandDisp(AM, IsBase, Op0,
510 Op1: cast<ConstantSDNode>(Val&: Op1)->getSExtValue());
511
512 if (IsBase && expandIndex(AM, Base: Op0, Index: Op1))
513 return true;
514 }
515 if (Opcode == SystemZISD::PCREL_OFFSET) {
516 SDValue Full = N.getOperand(i: 0);
517 SDValue Base = N.getOperand(i: 1);
518 SDValue Anchor = Base.getOperand(i: 0);
519 uint64_t Offset = (cast<GlobalAddressSDNode>(Val&: Full)->getOffset() -
520 cast<GlobalAddressSDNode>(Val&: Anchor)->getOffset());
521 return expandDisp(AM, IsBase, Op0: Base, Op1: Offset);
522 }
523 return false;
524}
525
526// Return true if an instruction with displacement range DR should be
527// used for displacement value Val. selectDisp(DR, Val) must already hold.
528static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
529 assert(selectDisp(DR, Val) && "Invalid displacement");
530 switch (DR) {
531 case SystemZAddressingMode::Disp12Only:
532 case SystemZAddressingMode::Disp20Only:
533 case SystemZAddressingMode::Disp20Only128:
534 return true;
535
536 case SystemZAddressingMode::Disp12Pair:
537 // Use the other instruction if the displacement is too large.
538 return isUInt<12>(x: Val);
539
540 case SystemZAddressingMode::Disp20Pair:
541 // Use the other instruction if the displacement is small enough.
542 return !isUInt<12>(x: Val);
543 }
544 llvm_unreachable("Unhandled displacement range");
545}
546
547// Return true if Base + Disp + Index should be performed by LA(Y).
548static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
549 // Don't use LA(Y) for constants.
550 if (!Base)
551 return false;
552
553 // Always use LA(Y) for frame addresses, since we know that the destination
554 // register is almost always (perhaps always) going to be different from
555 // the frame register.
556 if (Base->getOpcode() == ISD::FrameIndex)
557 return true;
558
559 if (Disp) {
560 // Always use LA(Y) if there is a base, displacement and index.
561 if (Index)
562 return true;
563
564 // Always use LA if the displacement is small enough. It should always
565 // be no worse than AGHI (and better if it avoids a move).
566 if (isUInt<12>(x: Disp))
567 return true;
568
569 // For similar reasons, always use LAY if the constant is too big for AGHI.
570 // LAY should be no worse than AGFI.
571 if (!isInt<16>(x: Disp))
572 return true;
573 } else {
574 // Don't use LA for plain registers.
575 if (!Index)
576 return false;
577
578 // Don't use LA for plain addition if the index operand is only used
579 // once. It should be a natural two-operand addition in that case.
580 if (Index->hasOneUse())
581 return false;
582
583 // Prefer addition if the second operation is sign-extended, in the
584 // hope of using AGF.
585 unsigned IndexOpcode = Index->getOpcode();
586 if (IndexOpcode == ISD::SIGN_EXTEND ||
587 IndexOpcode == ISD::SIGN_EXTEND_INREG)
588 return false;
589 }
590
591 // Don't use LA for two-operand addition if either operand is only
592 // used once. The addition instructions are better in that case.
593 if (Base->hasOneUse())
594 return false;
595
596 return true;
597}
598
599// Return true if Addr is suitable for AM, updating AM if so.
600bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
601 SystemZAddressingMode &AM) const {
602 // Start out assuming that the address will need to be loaded separately,
603 // then try to extend it as much as we can.
604 AM.Base = Addr;
605
606 // First try treating the address as a constant.
607 if (Addr.getOpcode() == ISD::Constant &&
608 expandDisp(AM, IsBase: true, Op0: SDValue(),
609 Op1: cast<ConstantSDNode>(Val&: Addr)->getSExtValue()))
610 ;
611 // Also see if it's a bare ADJDYNALLOC.
612 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
613 expandAdjDynAlloc(AM, IsBase: true, Value: SDValue()))
614 ;
615 else
616 // Otherwise try expanding each component.
617 while (expandAddress(AM, IsBase: true) ||
618 (AM.Index.getNode() && expandAddress(AM, IsBase: false)))
619 continue;
620
621 // Reject cases where it isn't profitable to use LA(Y).
622 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
623 !shouldUseLA(Base: AM.Base.getNode(), Disp: AM.Disp, Index: AM.Index.getNode()))
624 return false;
625
626 // Reject cases where the other instruction in a pair should be used.
627 if (!isValidDisp(DR: AM.DR, Val: AM.Disp))
628 return false;
629
630 // Make sure that ADJDYNALLOC is included where necessary.
631 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
632 return false;
633
634 LLVM_DEBUG(AM.dump(CurDAG));
635 return true;
636}
637
638// Insert a node into the DAG at least before Pos. This will reposition
639// the node as needed, and will assign it a node ID that is <= Pos's ID.
640// Note that this does *not* preserve the uniqueness of node IDs!
641// The selection DAG must no longer depend on their uniqueness when this
642// function is used.
643static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
644 if (N->getNodeId() == -1 ||
645 (SelectionDAGISel::getUninvalidatedNodeId(N: N.getNode()) >
646 SelectionDAGISel::getUninvalidatedNodeId(N: Pos))) {
647 DAG->RepositionNode(Position: Pos->getIterator(), N: N.getNode());
648 // Mark Node as invalid for pruning as after this it may be a successor to a
649 // selected node but otherwise be in the same position of Pos.
650 // Conservatively mark it with the same -abs(Id) to assure node id
651 // invariant is preserved.
652 N->setNodeId(Pos->getNodeId());
653 SelectionDAGISel::InvalidateNodeId(N: N.getNode());
654 }
655}
656
657void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
658 EVT VT, SDValue &Base,
659 SDValue &Disp) const {
660 Base = AM.Base;
661 if (!Base.getNode())
662 // Register 0 means "no base". This is mostly useful for shifts.
663 Base = CurDAG->getRegister(Reg: 0, VT);
664 else if (Base.getOpcode() == ISD::FrameIndex) {
665 // Lower a FrameIndex to a TargetFrameIndex.
666 int64_t FrameIndex = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
667 Base = CurDAG->getTargetFrameIndex(FI: FrameIndex, VT);
668 } else if (Base.getValueType() != VT) {
669 // Truncate values from i64 to i32, for shifts.
670 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
671 "Unexpected truncation");
672 SDLoc DL(Base);
673 SDValue Trunc = CurDAG->getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Base);
674 insertDAGNode(DAG: CurDAG, Pos: Base.getNode(), N: Trunc);
675 Base = Trunc;
676 }
677
678 // Lower the displacement to a TargetConstant.
679 Disp = CurDAG->getSignedTargetConstant(Val: AM.Disp, DL: SDLoc(Base), VT);
680}
681
682void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
683 EVT VT, SDValue &Base,
684 SDValue &Disp,
685 SDValue &Index) const {
686 getAddressOperands(AM, VT, Base, Disp);
687
688 Index = AM.Index;
689 if (!Index.getNode())
690 // Register 0 means "no index".
691 Index = CurDAG->getRegister(Reg: 0, VT);
692}
693
694bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
695 SDValue Addr, SDValue &Base,
696 SDValue &Disp) const {
697 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
698 if (!selectAddress(Addr, AM))
699 return false;
700
701 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp);
702 return true;
703}
704
705bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
706 SDValue Addr, SDValue &Base,
707 SDValue &Disp) const {
708 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
709 if (!selectAddress(Addr, AM) || AM.Index.getNode())
710 return false;
711
712 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp);
713 return true;
714}
715
716bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
717 SystemZAddressingMode::DispRange DR,
718 SDValue Addr, SDValue &Base,
719 SDValue &Disp, SDValue &Index) const {
720 SystemZAddressingMode AM(Form, DR);
721 if (!selectAddress(Addr, AM))
722 return false;
723
724 getAddressOperands(AM, VT: Addr.getValueType(), Base, Disp, Index);
725 return true;
726}
727
728bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
729 SDValue &Base,
730 SDValue &Disp,
731 SDValue &Index) const {
732 SDValue Regs[2];
733 if (selectBDXAddr12Only(Addr, Base&: Regs[0], Disp, Index&: Regs[1]) &&
734 Regs[0].getNode() && Regs[1].getNode()) {
735 for (unsigned int I = 0; I < 2; ++I) {
736 Base = Regs[I];
737 Index = Regs[1 - I];
738 // We can't tell here whether the index vector has the right type
739 // for the access; the caller needs to do that instead.
740 if (Index.getOpcode() == ISD::ZERO_EXTEND)
741 Index = Index.getOperand(i: 0);
742 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
743 Index.getOperand(i: 1) == Elem) {
744 Index = Index.getOperand(i: 0);
745 return true;
746 }
747 }
748 }
749 return false;
750}
751
752bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
753 uint64_t InsertMask) const {
754 // We're only interested in cases where the insertion is into some operand
755 // of Op, rather than into Op itself. The only useful case is an AND.
756 if (Op.getOpcode() != ISD::AND)
757 return false;
758
759 // We need a constant mask.
760 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1).getNode());
761 if (!MaskNode)
762 return false;
763
764 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
765 uint64_t AndMask = MaskNode->getZExtValue();
766 if (InsertMask & AndMask)
767 return false;
768
769 // It's only an insertion if all bits are covered or are known to be zero.
770 // The inner check covers all cases but is more expensive.
771 uint64_t Used = allOnes(Count: Op.getValueSizeInBits());
772 if (Used != (AndMask | InsertMask)) {
773 KnownBits Known = CurDAG->computeKnownBits(Op: Op.getOperand(i: 0));
774 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
775 return false;
776 }
777
778 Op = Op.getOperand(i: 0);
779 return true;
780}
781
782bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
783 uint64_t Mask) const {
784 const SystemZInstrInfo *TII = getInstrInfo();
785 if (RxSBG.Rotate != 0)
786 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
787 Mask &= RxSBG.Mask;
788 if (TII->isRxSBGMask(Mask, BitSize: RxSBG.BitSize, Start&: RxSBG.Start, End&: RxSBG.End)) {
789 RxSBG.Mask = Mask;
790 return true;
791 }
792 return false;
793}
794
795// Return true if any bits of (RxSBG.Input & Mask) are significant.
796static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
797 // Rotate the mask in the same way as RxSBG.Input is rotated.
798 if (RxSBG.Rotate != 0)
799 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
800 return (Mask & RxSBG.Mask) != 0;
801}
802
803bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
804 SDValue N = RxSBG.Input;
805 unsigned Opcode = N.getOpcode();
806 switch (Opcode) {
807 case ISD::TRUNCATE: {
808 if (RxSBG.Opcode == SystemZ::RNSBG)
809 return false;
810 if (N.getOperand(i: 0).getValueSizeInBits() > 64)
811 return false;
812 uint64_t BitSize = N.getValueSizeInBits();
813 uint64_t Mask = allOnes(Count: BitSize);
814 if (!refineRxSBGMask(RxSBG, Mask))
815 return false;
816 RxSBG.Input = N.getOperand(i: 0);
817 return true;
818 }
819 case ISD::AND: {
820 if (RxSBG.Opcode == SystemZ::RNSBG)
821 return false;
822
823 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
824 if (!MaskNode)
825 return false;
826
827 SDValue Input = N.getOperand(i: 0);
828 uint64_t Mask = MaskNode->getZExtValue();
829 if (!refineRxSBGMask(RxSBG, Mask)) {
830 // If some bits of Input are already known zeros, those bits will have
831 // been removed from the mask. See if adding them back in makes the
832 // mask suitable.
833 KnownBits Known = CurDAG->computeKnownBits(Op: Input);
834 Mask |= Known.Zero.getZExtValue();
835 if (!refineRxSBGMask(RxSBG, Mask))
836 return false;
837 }
838 RxSBG.Input = Input;
839 return true;
840 }
841
842 case ISD::OR: {
843 if (RxSBG.Opcode != SystemZ::RNSBG)
844 return false;
845
846 auto *MaskNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
847 if (!MaskNode)
848 return false;
849
850 SDValue Input = N.getOperand(i: 0);
851 uint64_t Mask = ~MaskNode->getZExtValue();
852 if (!refineRxSBGMask(RxSBG, Mask)) {
853 // If some bits of Input are already known ones, those bits will have
854 // been removed from the mask. See if adding them back in makes the
855 // mask suitable.
856 KnownBits Known = CurDAG->computeKnownBits(Op: Input);
857 Mask &= ~Known.One.getZExtValue();
858 if (!refineRxSBGMask(RxSBG, Mask))
859 return false;
860 }
861 RxSBG.Input = Input;
862 return true;
863 }
864
865 case ISD::ROTL: {
866 // Any 64-bit rotate left can be merged into the RxSBG.
867 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
868 return false;
869 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
870 if (!CountNode)
871 return false;
872
873 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
874 RxSBG.Input = N.getOperand(i: 0);
875 return true;
876 }
877
878 case ISD::ANY_EXTEND:
879 // Bits above the extended operand are don't-care.
880 RxSBG.Input = N.getOperand(i: 0);
881 return true;
882
883 case ISD::ZERO_EXTEND:
884 if (RxSBG.Opcode != SystemZ::RNSBG) {
885 // Restrict the mask to the extended operand.
886 unsigned InnerBitSize = N.getOperand(i: 0).getValueSizeInBits();
887 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: InnerBitSize)))
888 return false;
889
890 RxSBG.Input = N.getOperand(i: 0);
891 return true;
892 }
893 [[fallthrough]];
894
895 case ISD::SIGN_EXTEND: {
896 // Check that the extension bits are don't-care (i.e. are masked out
897 // by the final mask).
898 unsigned BitSize = N.getValueSizeInBits();
899 unsigned InnerBitSize = N.getOperand(i: 0).getValueSizeInBits();
900 if (maskMatters(RxSBG, Mask: allOnes(Count: BitSize) - allOnes(Count: InnerBitSize))) {
901 // In the case where only the sign bit is active, increase Rotate with
902 // the extension width.
903 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
904 RxSBG.Rotate += (BitSize - InnerBitSize);
905 else
906 return false;
907 }
908
909 RxSBG.Input = N.getOperand(i: 0);
910 return true;
911 }
912
913 case ISD::SHL: {
914 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
915 if (!CountNode)
916 return false;
917
918 uint64_t Count = CountNode->getZExtValue();
919 unsigned BitSize = N.getValueSizeInBits();
920 if (Count < 1 || Count >= BitSize)
921 return false;
922
923 if (RxSBG.Opcode == SystemZ::RNSBG) {
924 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
925 // count bits from RxSBG.Input are ignored.
926 if (maskMatters(RxSBG, Mask: allOnes(Count)))
927 return false;
928 } else {
929 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
930 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: BitSize - Count) << Count))
931 return false;
932 }
933
934 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
935 RxSBG.Input = N.getOperand(i: 0);
936 return true;
937 }
938
939 case ISD::SRL:
940 case ISD::SRA: {
941 auto *CountNode = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1).getNode());
942 if (!CountNode)
943 return false;
944
945 uint64_t Count = CountNode->getZExtValue();
946 unsigned BitSize = N.getValueSizeInBits();
947 if (Count < 1 || Count >= BitSize)
948 return false;
949
950 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
951 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
952 // count bits from RxSBG.Input are ignored.
953 if (maskMatters(RxSBG, Mask: allOnes(Count) << (BitSize - Count)))
954 return false;
955 } else {
956 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
957 // which is similar to SLL above.
958 if (!refineRxSBGMask(RxSBG, Mask: allOnes(Count: BitSize - Count)))
959 return false;
960 }
961
962 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
963 RxSBG.Input = N.getOperand(i: 0);
964 return true;
965 }
966 default:
967 return false;
968 }
969}
970
971SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
972 SDNode *N = CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
973 return SDValue(N, 0);
974}
975
976SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
977 SDValue N) const {
978 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
979 return CurDAG->getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32,
980 DL, VT, Operand: getUNDEF(DL, VT: MVT::i64), Subreg: N);
981 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
982 return CurDAG->getTargetExtractSubreg(SRIdx: SystemZ::subreg_l32, DL, VT, Operand: N);
983 assert(N.getValueType() == VT && "Unexpected value types");
984 return N;
985}
986
987bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
988 SDLoc DL(N);
989 EVT VT = N->getValueType(ResNo: 0);
990 if (!VT.isInteger() || VT.getSizeInBits() > 64)
991 return false;
992 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
993 unsigned Count = 0;
994 while (expandRxSBG(RxSBG&: RISBG))
995 // The widening or narrowing is expected to be free.
996 // Counting widening or narrowing as a saved operation will result in
997 // preferring an R*SBG over a simple shift/logical instruction.
998 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
999 RISBG.Input.getOpcode() != ISD::TRUNCATE)
1000 Count += 1;
1001 if (Count == 0 || isa<ConstantSDNode>(Val: RISBG.Input))
1002 return false;
1003
1004 // Prefer to use normal shift instructions over RISBG, since they can handle
1005 // all cases and are sometimes shorter.
1006 if (Count == 1 && N->getOpcode() != ISD::AND)
1007 return false;
1008
1009 // Prefer LOAD LOGICAL INDEXED ADDRESS over RISBG in the case where we
1010 // can use its displacement to pull in an addition.
1011 if (Subtarget->hasMiscellaneousExtensions4() &&
1012 RISBG.Rotate >= 1 && RISBG.Rotate <= 4 &&
1013 RISBG.Mask == (((uint64_t)1 << 32) - 1) << RISBG.Rotate &&
1014 RISBG.Input.getOpcode() == ISD::ADD)
1015 if (auto *C = dyn_cast<ConstantSDNode>(Val: RISBG.Input.getOperand(i: 1)))
1016 if (isInt<20>(x: C->getSExtValue()))
1017 return false;
1018
1019 // Prefer register extensions like LLC over RISBG. Also prefer to start
1020 // out with normal ANDs if one instruction would be enough. We can convert
1021 // these ANDs into an RISBG later if a three-address instruction is useful.
1022 if (RISBG.Rotate == 0) {
1023 bool PreferAnd = false;
1024 // Prefer AND for any 32-bit and-immediate operation.
1025 if (VT == MVT::i32)
1026 PreferAnd = true;
1027 // As well as for any 64-bit operation that can be implemented via LLC(R),
1028 // LLH(R), LLGT(R), or one of the and-immediate instructions.
1029 else if (RISBG.Mask == 0xff ||
1030 RISBG.Mask == 0xffff ||
1031 RISBG.Mask == 0x7fffffff ||
1032 SystemZ::isImmLF(Val: ~RISBG.Mask) ||
1033 SystemZ::isImmHF(Val: ~RISBG.Mask))
1034 PreferAnd = true;
1035 // And likewise for the LLZRGF instruction, which doesn't have a register
1036 // to register version.
1037 else if (auto *Load = dyn_cast<LoadSDNode>(Val&: RISBG.Input)) {
1038 if (Load->getMemoryVT() == MVT::i32 &&
1039 (Load->getExtensionType() == ISD::EXTLOAD ||
1040 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1041 RISBG.Mask == 0xffffff00 &&
1042 Subtarget->hasLoadAndZeroRightmostByte())
1043 PreferAnd = true;
1044 }
1045 if (PreferAnd) {
1046 // Replace the current node with an AND. Note that the current node
1047 // might already be that same AND, in which case it is already CSE'd
1048 // with it, and we must not call ReplaceNode.
1049 SDValue In = convertTo(DL, VT, N: RISBG.Input);
1050 SDValue Mask = CurDAG->getConstant(Val: RISBG.Mask, DL, VT);
1051 SDValue New = CurDAG->getNode(Opcode: ISD::AND, DL, VT, N1: In, N2: Mask);
1052 if (N != New.getNode()) {
1053 insertDAGNode(DAG: CurDAG, Pos: N, N: Mask);
1054 insertDAGNode(DAG: CurDAG, Pos: N, N: New);
1055 ReplaceNode(F: N, T: New.getNode());
1056 N = New.getNode();
1057 }
1058 // Now, select the machine opcode to implement this operation.
1059 if (!N->isMachineOpcode())
1060 SelectCode(N);
1061 return true;
1062 }
1063 }
1064
1065 unsigned Opcode = SystemZ::RISBG;
1066 // Prefer RISBGN if available, since it does not clobber CC.
1067 if (Subtarget->hasMiscellaneousExtensions())
1068 Opcode = SystemZ::RISBGN;
1069 EVT OpcodeVT = MVT::i64;
1070 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1071 // We can only use the 32-bit instructions if all source bits are
1072 // in the low 32 bits without wrapping, both after rotation (because
1073 // of the smaller range for Start and End) and before rotation
1074 // (because the input value is truncated).
1075 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1076 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1077 ((RISBG.End + RISBG.Rotate) & 63) >=
1078 ((RISBG.Start + RISBG.Rotate) & 63)) {
1079 Opcode = SystemZ::RISBMux;
1080 OpcodeVT = MVT::i32;
1081 RISBG.Start &= 31;
1082 RISBG.End &= 31;
1083 }
1084 SDValue Ops[5] = {
1085 getUNDEF(DL, VT: OpcodeVT),
1086 convertTo(DL, VT: OpcodeVT, N: RISBG.Input),
1087 CurDAG->getTargetConstant(Val: RISBG.Start, DL, VT: MVT::i32),
1088 CurDAG->getTargetConstant(Val: RISBG.End | 128, DL, VT: MVT::i32),
1089 CurDAG->getTargetConstant(Val: RISBG.Rotate, DL, VT: MVT::i32)
1090 };
1091 SDValue New = convertTo(
1092 DL, VT, N: SDValue(CurDAG->getMachineNode(Opcode, dl: DL, VT: OpcodeVT, Ops), 0));
1093 ReplaceNode(F: N, T: New.getNode());
1094 return true;
1095}
1096
1097bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1098 SDLoc DL(N);
1099 EVT VT = N->getValueType(ResNo: 0);
1100 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1101 return false;
1102 // Try treating each operand of N as the second operand of the RxSBG
1103 // and see which goes deepest.
1104 RxSBGOperands RxSBG[] = {
1105 RxSBGOperands(Opcode, N->getOperand(Num: 0)),
1106 RxSBGOperands(Opcode, N->getOperand(Num: 1))
1107 };
1108 unsigned Count[] = { 0, 0 };
1109 for (unsigned I = 0; I < 2; ++I)
1110 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG&: RxSBG[I]))
1111 // In cases of multiple users it seems better to keep the simple
1112 // instruction as they are one cycle faster, and it also helps in cases
1113 // where both inputs share a common node.
1114 // The widening or narrowing is expected to be free. Counting widening
1115 // or narrowing as a saved operation will result in preferring an R*SBG
1116 // over a simple shift/logical instruction.
1117 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1118 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1119 Count[I] += 1;
1120
1121 // Do nothing if neither operand is suitable.
1122 if (Count[0] == 0 && Count[1] == 0)
1123 return false;
1124
1125 // Pick the deepest second operand.
1126 unsigned I = Count[0] > Count[1] ? 0 : 1;
1127 SDValue Op0 = N->getOperand(Num: I ^ 1);
1128
1129 // Prefer IC for character insertions from memory.
1130 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1131 if (auto *Load = dyn_cast<LoadSDNode>(Val: Op0.getNode()))
1132 if (Load->getMemoryVT() == MVT::i8)
1133 return false;
1134
1135 // See whether we can avoid an AND in the first operand by converting
1136 // ROSBG to RISBG.
1137 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op&: Op0, InsertMask: RxSBG[I].Mask)) {
1138 Opcode = SystemZ::RISBG;
1139 // Prefer RISBGN if available, since it does not clobber CC.
1140 if (Subtarget->hasMiscellaneousExtensions())
1141 Opcode = SystemZ::RISBGN;
1142 }
1143
1144 SDValue Ops[5] = {
1145 convertTo(DL, VT: MVT::i64, N: Op0),
1146 convertTo(DL, VT: MVT::i64, N: RxSBG[I].Input),
1147 CurDAG->getTargetConstant(Val: RxSBG[I].Start, DL, VT: MVT::i32),
1148 CurDAG->getTargetConstant(Val: RxSBG[I].End, DL, VT: MVT::i32),
1149 CurDAG->getTargetConstant(Val: RxSBG[I].Rotate, DL, VT: MVT::i32)
1150 };
1151 SDValue New = convertTo(
1152 DL, VT, N: SDValue(CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::i64, Ops), 0));
1153 ReplaceNode(F: N, T: New.getNode());
1154 return true;
1155}
1156
1157void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1158 SDValue Op0, uint64_t UpperVal,
1159 uint64_t LowerVal) {
1160 EVT VT = Node->getValueType(ResNo: 0);
1161 SDLoc DL(Node);
1162 SDValue Upper = CurDAG->getConstant(Val: UpperVal, DL, VT);
1163 if (Op0.getNode())
1164 Upper = CurDAG->getNode(Opcode, DL, VT, N1: Op0, N2: Upper);
1165
1166 {
1167 // When we haven't passed in Op0, Upper will be a constant. In order to
1168 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1169 // SelectCode first and end up with an opaque machine node. This means that
1170 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1171 // SelectCode.
1172 //
1173 // Note that in the case where Op0 is passed in we could just call
1174 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1175 // the handle at all, but it's fine to do it here.
1176 //
1177 // TODO: This is a pretty hacky way to do this. Can we do something that
1178 // doesn't require a two paragraph explanation?
1179 HandleSDNode Handle(Upper);
1180 SelectCode(N: Upper.getNode());
1181 Upper = Handle.getValue();
1182 }
1183
1184 SDValue Lower = CurDAG->getConstant(Val: LowerVal, DL, VT);
1185 SDValue Or = CurDAG->getNode(Opcode, DL, VT, N1: Upper, N2: Lower);
1186
1187 ReplaceNode(F: Node, T: Or.getNode());
1188
1189 SelectCode(N: Or.getNode());
1190}
1191
1192void SystemZDAGToDAGISel::loadVectorConstant(
1193 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1194 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1195 VCI.Opcode == SystemZISD::REPLICATE ||
1196 VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1197 "Bad opcode!");
1198 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1199 EVT VT = Node->getValueType(ResNo: 0);
1200 SDLoc DL(Node);
1201 SmallVector<SDValue, 2> Ops;
1202 for (unsigned OpVal : VCI.OpVals)
1203 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: OpVal, DL, VT: MVT::i32));
1204 SDValue Op = CurDAG->getNode(Opcode: VCI.Opcode, DL, VT: VCI.VecVT, Ops);
1205
1206 if (VCI.VecVT == VT.getSimpleVT())
1207 ReplaceNode(F: Node, T: Op.getNode());
1208 else if (VT.getSizeInBits() == 128) {
1209 SDValue BitCast = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
1210 ReplaceNode(F: Node, T: BitCast.getNode());
1211 SelectCode(N: BitCast.getNode());
1212 } else { // half, float or double
1213 unsigned SubRegIdx = (VT.getSizeInBits() == 16 ? SystemZ::subreg_h16
1214 : VT.getSizeInBits() == 32 ? SystemZ::subreg_h32
1215 : SystemZ::subreg_h64);
1216 ReplaceNode(
1217 F: Node, T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: Op).getNode());
1218 }
1219 SelectCode(N: Op.getNode());
1220}
1221
1222SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {
1223 SDNode *ResNode;
1224 assert (VT.getSizeInBits() == 128);
1225
1226 SDValue CP = CurDAG->getTargetConstantPool(
1227 C: ConstantInt::get(Ty: Type::getInt128Ty(C&: *CurDAG->getContext()), V: Val),
1228 VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1229
1230 EVT PtrVT = CP.getValueType();
1231 SDValue Ops[] = {
1232 SDValue(CurDAG->getMachineNode(Opcode: SystemZ::LARL, dl: DL, VT: PtrVT, Op1: CP), 0),
1233 CurDAG->getTargetConstant(Val: 0, DL, VT: PtrVT),
1234 CurDAG->getRegister(Reg: 0, VT: PtrVT),
1235 CurDAG->getEntryNode()
1236 };
1237 ResNode = CurDAG->getMachineNode(Opcode: SystemZ::VL, dl: DL, VT1: VT, VT2: MVT::Other, Ops);
1238
1239 // Annotate ResNode with memory operand information so that MachineInstr
1240 // queries work properly. This e.g. gives the register allocation the
1241 // required information for rematerialization.
1242 MachineFunction& MF = CurDAG->getMachineFunction();
1243 MachineMemOperand *MemOp =
1244 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
1245 F: MachineMemOperand::MOLoad, Size: 16, BaseAlignment: Align(8));
1246
1247 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: ResNode), NewMemRefs: {MemOp});
1248 return ResNode;
1249}
1250
1251bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1252 SDValue ElemV = N->getOperand(Num: 2);
1253 auto *ElemN = dyn_cast<ConstantSDNode>(Val&: ElemV);
1254 if (!ElemN)
1255 return false;
1256
1257 unsigned Elem = ElemN->getZExtValue();
1258 EVT VT = N->getValueType(ResNo: 0);
1259 if (Elem >= VT.getVectorNumElements())
1260 return false;
1261
1262 auto *Load = dyn_cast<LoadSDNode>(Val: N->getOperand(Num: 1));
1263 if (!Load || !Load->hasNUsesOfValue(NUses: 1, Value: 0))
1264 return false;
1265 if (Load->getMemoryVT().getSizeInBits() !=
1266 Load->getValueType(ResNo: 0).getSizeInBits())
1267 return false;
1268
1269 SDValue Base, Disp, Index;
1270 if (!selectBDVAddr12Only(Addr: Load->getBasePtr(), Elem: ElemV, Base, Disp, Index) ||
1271 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1272 return false;
1273
1274 SDLoc DL(Load);
1275 SDValue Ops[] = {
1276 N->getOperand(Num: 0), Base, Disp, Index,
1277 CurDAG->getTargetConstant(Val: Elem, DL, VT: MVT::i32), Load->getChain()
1278 };
1279 SDNode *Res = CurDAG->getMachineNode(Opcode, dl: DL, VT1: VT, VT2: MVT::Other, Ops);
1280 ReplaceUses(F: SDValue(Load, 1), T: SDValue(Res, 1));
1281 ReplaceNode(F: N, T: Res);
1282 return true;
1283}
1284
1285bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1286 SDValue Value = Store->getValue();
1287 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1288 return false;
1289 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1290 return false;
1291
1292 SDValue ElemV = Value.getOperand(i: 1);
1293 auto *ElemN = dyn_cast<ConstantSDNode>(Val&: ElemV);
1294 if (!ElemN)
1295 return false;
1296
1297 SDValue Vec = Value.getOperand(i: 0);
1298 EVT VT = Vec.getValueType();
1299 unsigned Elem = ElemN->getZExtValue();
1300 if (Elem >= VT.getVectorNumElements())
1301 return false;
1302
1303 SDValue Base, Disp, Index;
1304 if (!selectBDVAddr12Only(Addr: Store->getBasePtr(), Elem: ElemV, Base, Disp, Index) ||
1305 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1306 return false;
1307
1308 SDLoc DL(Store);
1309 SDValue Ops[] = {
1310 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Val: Elem, DL, VT: MVT::i32),
1311 Store->getChain()
1312 };
1313 ReplaceNode(F: Store, T: CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Other, Ops));
1314 return true;
1315}
1316
1317// Check whether or not the chain ending in StoreNode is suitable for doing
1318// the {load; op; store} to modify transformation.
1319static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1320 SDValue StoredVal, SelectionDAG *CurDAG,
1321 LoadSDNode *&LoadNode,
1322 SDValue &InputChain) {
1323 // Is the stored value result 0 of the operation?
1324 if (StoredVal.getResNo() != 0)
1325 return false;
1326
1327 // Are there other uses of the loaded value than the operation?
1328 if (!StoredVal.getNode()->hasNUsesOfValue(NUses: 1, Value: 0))
1329 return false;
1330
1331 // Is the store non-extending and non-indexed?
1332 if (!ISD::isNormalStore(N: StoreNode) || StoreNode->isNonTemporal())
1333 return false;
1334
1335 SDValue Load = StoredVal->getOperand(Num: 0);
1336 // Is the stored value a non-extending and non-indexed load?
1337 if (!ISD::isNormalLoad(N: Load.getNode()))
1338 return false;
1339
1340 // Return LoadNode by reference.
1341 LoadNode = cast<LoadSDNode>(Val&: Load);
1342
1343 // Is store the only read of the loaded value?
1344 if (!Load.hasOneUse())
1345 return false;
1346
1347 // Is the address of the store the same as the load?
1348 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1349 LoadNode->getOffset() != StoreNode->getOffset())
1350 return false;
1351
1352 // Check if the chain is produced by the load or is a TokenFactor with
1353 // the load output chain as an operand. Return InputChain by reference.
1354 SDValue Chain = StoreNode->getChain();
1355
1356 bool ChainCheck = false;
1357 if (Chain == Load.getValue(R: 1)) {
1358 ChainCheck = true;
1359 InputChain = LoadNode->getChain();
1360 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1361 SmallVector<SDValue, 4> ChainOps;
1362 SmallVector<const SDNode *, 4> LoopWorklist;
1363 SmallPtrSet<const SDNode *, 16> Visited;
1364 const unsigned int Max = 1024;
1365 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1366 SDValue Op = Chain.getOperand(i);
1367 if (Op == Load.getValue(R: 1)) {
1368 ChainCheck = true;
1369 // Drop Load, but keep its chain. No cycle check necessary.
1370 ChainOps.push_back(Elt: Load.getOperand(i: 0));
1371 continue;
1372 }
1373 LoopWorklist.push_back(Elt: Op.getNode());
1374 ChainOps.push_back(Elt: Op);
1375 }
1376
1377 if (ChainCheck) {
1378 // Add the other operand of StoredVal to worklist.
1379 for (SDValue Op : StoredVal->ops())
1380 if (Op.getNode() != LoadNode)
1381 LoopWorklist.push_back(Elt: Op.getNode());
1382
1383 // Check if Load is reachable from any of the nodes in the worklist.
1384 if (SDNode::hasPredecessorHelper(N: Load.getNode(), Visited, Worklist&: LoopWorklist, MaxSteps: Max,
1385 TopologicalPrune: true))
1386 return false;
1387
1388 // Make a new TokenFactor with all the other input chains except
1389 // for the load.
1390 InputChain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain),
1391 VT: MVT::Other, Ops: ChainOps);
1392 }
1393 }
1394 if (!ChainCheck)
1395 return false;
1396
1397 return true;
1398}
1399
1400// Change a chain of {load; op; store} of the same value into a simple op
1401// through memory of that value, if the uses of the modified value and its
1402// address are suitable.
1403//
1404// The tablegen pattern memory operand pattern is currently not able to match
1405// the case where the CC on the original operation are used.
1406//
1407// See the equivalent routine in X86ISelDAGToDAG for further comments.
1408bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1409 StoreSDNode *StoreNode = cast<StoreSDNode>(Val: Node);
1410 SDValue StoredVal = StoreNode->getOperand(Num: 1);
1411 unsigned Opc = StoredVal->getOpcode();
1412 SDLoc DL(StoreNode);
1413
1414 // Before we try to select anything, make sure this is memory operand size
1415 // and opcode we can handle. Note that this must match the code below that
1416 // actually lowers the opcodes.
1417 EVT MemVT = StoreNode->getMemoryVT();
1418 unsigned NewOpc = 0;
1419 bool NegateOperand = false;
1420 switch (Opc) {
1421 default:
1422 return false;
1423 case SystemZISD::SSUBO:
1424 NegateOperand = true;
1425 [[fallthrough]];
1426 case SystemZISD::SADDO:
1427 if (MemVT == MVT::i32)
1428 NewOpc = SystemZ::ASI;
1429 else if (MemVT == MVT::i64)
1430 NewOpc = SystemZ::AGSI;
1431 else
1432 return false;
1433 break;
1434 case SystemZISD::USUBO:
1435 NegateOperand = true;
1436 [[fallthrough]];
1437 case SystemZISD::UADDO:
1438 if (MemVT == MVT::i32)
1439 NewOpc = SystemZ::ALSI;
1440 else if (MemVT == MVT::i64)
1441 NewOpc = SystemZ::ALGSI;
1442 else
1443 return false;
1444 break;
1445 }
1446
1447 LoadSDNode *LoadNode = nullptr;
1448 SDValue InputChain;
1449 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1450 InputChain))
1451 return false;
1452
1453 SDValue Operand = StoredVal.getOperand(i: 1);
1454 auto *OperandC = dyn_cast<ConstantSDNode>(Val&: Operand);
1455 if (!OperandC)
1456 return false;
1457 auto OperandV = OperandC->getAPIntValue();
1458 if (NegateOperand)
1459 OperandV = -OperandV;
1460 if (OperandV.getSignificantBits() > 8)
1461 return false;
1462 Operand = CurDAG->getTargetConstant(Val: OperandV, DL, VT: MemVT);
1463
1464 SDValue Base, Disp;
1465 if (!selectBDAddr20Only(Addr: StoreNode->getBasePtr(), Base, Disp))
1466 return false;
1467
1468 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1469 MachineSDNode *Result =
1470 CurDAG->getMachineNode(Opcode: NewOpc, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
1471 CurDAG->setNodeMemRefs(
1472 N: Result, NewMemRefs: {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1473
1474 ReplaceUses(F: SDValue(StoreNode, 0), T: SDValue(Result, 1));
1475 ReplaceUses(F: SDValue(StoredVal.getNode(), 1), T: SDValue(Result, 0));
1476 CurDAG->RemoveDeadNode(N: Node);
1477 return true;
1478}
1479
1480bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1481 LoadSDNode *Load) const {
1482 // Check that the two memory operands have the same size.
1483 if (Load->getMemoryVT() != Store->getMemoryVT())
1484 return false;
1485
1486 // Volatility stops an access from being decomposed.
1487 if (Load->isVolatile() || Store->isVolatile())
1488 return false;
1489
1490 // There's no chance of overlap if the load is invariant.
1491 if (Load->isInvariant() && Load->isDereferenceable())
1492 return true;
1493
1494 // Otherwise we need to check whether there's an alias.
1495 const Value *V1 = Load->getMemOperand()->getValue();
1496 const Value *V2 = Store->getMemOperand()->getValue();
1497 if (!V1 || !V2)
1498 return false;
1499
1500 // Reject equality.
1501 uint64_t Size = Load->getMemoryVT().getStoreSize();
1502 int64_t End1 = Load->getSrcValueOffset() + Size;
1503 int64_t End2 = Store->getSrcValueOffset() + Size;
1504 if (V1 == V2 && End1 == End2)
1505 return false;
1506
1507 return BatchAA->isNoAlias(LocA: MemoryLocation(V1, End1, Load->getAAInfo()),
1508 LocB: MemoryLocation(V2, End2, Store->getAAInfo()));
1509}
1510
1511bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1512 auto *Store = cast<StoreSDNode>(Val: N);
1513 auto *Load = cast<LoadSDNode>(Val: Store->getValue());
1514
1515 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1516 // instructions.
1517 uint64_t Size = Load->getMemoryVT().getStoreSize();
1518 if (Size > 1 && Size <= 8) {
1519 // Prefer LHRL, LRL and LGRL.
1520 if (SystemZISD::isPCREL(Opcode: Load->getBasePtr().getOpcode()))
1521 return false;
1522 // Prefer STHRL, STRL and STGRL.
1523 if (SystemZISD::isPCREL(Opcode: Store->getBasePtr().getOpcode()))
1524 return false;
1525 }
1526
1527 return canUseBlockOperation(Store, Load);
1528}
1529
1530bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1531 unsigned I) const {
1532 auto *StoreA = cast<StoreSDNode>(Val: N);
1533 auto *LoadA = cast<LoadSDNode>(Val: StoreA->getValue().getOperand(i: 1 - I));
1534 auto *LoadB = cast<LoadSDNode>(Val: StoreA->getValue().getOperand(i: I));
1535 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1536 canUseBlockOperation(Store: StoreA, Load: LoadB);
1537}
1538
1539bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1540
1541 auto *MemAccess = cast<MemSDNode>(Val: N);
1542 auto *LdSt = dyn_cast<LSBaseSDNode>(Val: MemAccess);
1543 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1544 SDValue BasePtr = MemAccess->getBasePtr();
1545 MachineMemOperand *MMO = MemAccess->getMemOperand();
1546 assert(MMO && "Expected a memory operand.");
1547
1548 // The memory access must have a proper alignment and no index register.
1549 // Only load and store nodes have the offset operand (atomic loads do not).
1550 if (MemAccess->getAlign().value() < StoreSize ||
1551 (LdSt && !LdSt->getOffset().isUndef()))
1552 return false;
1553
1554 // The MMO must not have an unaligned offset.
1555 if (MMO->getOffset() % StoreSize != 0)
1556 return false;
1557
1558 // An access to GOT or the Constant Pool is aligned.
1559 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1560 if ((PSV->isGOT() || PSV->isConstantPool()))
1561 return true;
1562
1563 // Check the alignment of a Global Address.
1564 if (BasePtr.getNumOperands())
1565 if (GlobalAddressSDNode *GA =
1566 dyn_cast<GlobalAddressSDNode>(Val: BasePtr.getOperand(i: 0))) {
1567 // The immediate offset must be aligned.
1568 if (GA->getOffset() % StoreSize != 0)
1569 return false;
1570
1571 // The alignment of the symbol itself must be at least the store size.
1572 const GlobalValue *GV = GA->getGlobal();
1573 const DataLayout &DL = GV->getDataLayout();
1574 if (GV->getPointerAlignment(DL).value() < StoreSize)
1575 return false;
1576 }
1577
1578 return true;
1579}
1580
1581ISD::LoadExtType SystemZDAGToDAGISel::getLoadExtType(SDNode *N) const {
1582 ISD::LoadExtType ETy;
1583 if (auto *L = dyn_cast<LoadSDNode>(Val: N))
1584 ETy = L->getExtensionType();
1585 else if (auto *AL = dyn_cast<AtomicSDNode>(Val: N))
1586 ETy = AL->getExtensionType();
1587 else
1588 llvm_unreachable("Unkown load node type.");
1589 return ETy;
1590}
1591
1592void SystemZDAGToDAGISel::Select(SDNode *Node) {
1593 // If we have a custom node, we already have selected!
1594 if (Node->isMachineOpcode()) {
1595 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1596 Node->setNodeId(-1);
1597 return;
1598 }
1599
1600 unsigned Opcode = Node->getOpcode();
1601 switch (Opcode) {
1602 case ISD::OR:
1603 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant)
1604 if (tryRxSBG(N: Node, Opcode: SystemZ::ROSBG))
1605 return;
1606 goto or_xor;
1607
1608 case ISD::XOR:
1609 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant)
1610 if (tryRxSBG(N: Node, Opcode: SystemZ::RXSBG))
1611 return;
1612 // Fall through.
1613 or_xor:
1614 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1615 // split the operation into two. If both operands here happen to be
1616 // constant, leave this to common code to optimize.
1617 if (Node->getValueType(ResNo: 0) == MVT::i64 &&
1618 Node->getOperand(Num: 0).getOpcode() != ISD::Constant)
1619 if (auto *Op1 = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
1620 uint64_t Val = Op1->getZExtValue();
1621 // Don't split the operation if we can match one of the combined
1622 // logical operations provided by miscellaneous-extensions-3.
1623 if (Subtarget->hasMiscellaneousExtensions3()) {
1624 unsigned ChildOpcode = Node->getOperand(Num: 0).getOpcode();
1625 // Check whether this expression matches NAND/NOR/NXOR.
1626 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1627 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1628 ChildOpcode == ISD::XOR)
1629 break;
1630 // Check whether this expression matches OR-with-complement
1631 // (or matches an alternate pattern for NXOR).
1632 if (ChildOpcode == ISD::XOR) {
1633 auto Op0 = Node->getOperand(Num: 0);
1634 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Val: Op0->getOperand(Num: 1)))
1635 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1636 break;
1637 }
1638 }
1639 // Don't split an XOR with -1 as LCGR/AGHI is more compact.
1640 if (Opcode == ISD::XOR && Op1->isAllOnes())
1641 break;
1642 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1643 splitLargeImmediate(Opcode, Node, Op0: Node->getOperand(Num: 0),
1644 UpperVal: Val - uint32_t(Val), LowerVal: uint32_t(Val));
1645 return;
1646 }
1647 }
1648 break;
1649
1650 case ISD::AND:
1651 if (Node->getOperand(Num: 1).getOpcode() != ISD::Constant) {
1652 if (tryRxSBG(N: Node, Opcode: SystemZ::RNSBG))
1653 return;
1654 } else {
1655 // Use patterns for zero-extending of vector element extraction.
1656 if (Node->getValueType(ResNo: 0) == MVT::i64 &&
1657 Node->getOperand(Num: 0)->getOpcode() == ISD::ANY_EXTEND) {
1658 SDValue Input = Node->getOperand(Num: 0)->getOperand(Num: 0);
1659 uint64_t Mask =
1660 cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))->getZExtValue();
1661 if (Input->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1662 EVT VecVT = Input->getOperand(Num: 0)->getValueType(ResNo: 0);
1663 unsigned EltBits = VecVT.getScalarSizeInBits();
1664 if (allOnes(Count: EltBits) == Mask)
1665 break;
1666 }
1667 }
1668 }
1669
1670 [[fallthrough]];
1671 case ISD::ROTL:
1672 case ISD::SHL:
1673 case ISD::SRL:
1674 case ISD::ZERO_EXTEND:
1675 if (tryRISBGZero(N: Node))
1676 return;
1677 break;
1678
1679 case ISD::BSWAP:
1680 if (Node->getValueType(ResNo: 0) == MVT::i128) {
1681 SDLoc DL(Node);
1682 SDValue Src = Node->getOperand(Num: 0);
1683 Src = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Src);
1684
1685 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1686 SDNode *Mask = loadPoolVectorConstant(Val: APInt(128, Bytes), VT: MVT::v16i8, DL);
1687 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) };
1688 SDValue Res = SDValue(CurDAG->getMachineNode(Opcode: SystemZ::VPERM, dl: DL,
1689 VT: MVT::v16i8, Ops), 0);
1690
1691 Res = CurDAG->getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i128, Operand: Res);
1692 SDNode *ResNode = Res.getNode();
1693 ReplaceNode(F: Node, T: ResNode);
1694 SelectCode(N: Src.getNode());
1695 SelectCode(N: ResNode);
1696 return;
1697 }
1698 break;
1699
1700 case ISD::Constant:
1701 // If this is a 64-bit constant that is out of the range of LLILF,
1702 // LLIHF and LGFI, split it into two 32-bit pieces.
1703 if (Node->getValueType(ResNo: 0) == MVT::i64) {
1704 uint64_t Val = Node->getAsZExtVal();
1705 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(x: Val)) {
1706 splitLargeImmediate(Opcode: ISD::OR, Node, Op0: SDValue(), UpperVal: Val - uint32_t(Val),
1707 LowerVal: uint32_t(Val));
1708 return;
1709 }
1710 }
1711 if (Node->getValueType(ResNo: 0) == MVT::i128) {
1712 const APInt &Val = Node->getAsAPIntVal();
1713 SystemZVectorConstantInfo VCI(Val);
1714 if (VCI.isVectorConstantLegal(Subtarget: *Subtarget)) {
1715 loadVectorConstant(VCI, Node);
1716 return;
1717 }
1718 // If we can't materialize the constant we need to use a literal pool.
1719 SDNode *ResNode = loadPoolVectorConstant(Val, VT: MVT::i128, DL: SDLoc(Node));
1720 ReplaceNode(F: Node, T: ResNode);
1721 return;
1722 }
1723 break;
1724
1725 case SystemZISD::SELECT_CCMASK: {
1726 SDValue Op0 = Node->getOperand(Num: 0);
1727 SDValue Op1 = Node->getOperand(Num: 1);
1728 // Prefer to put any load first, so that it can be matched as a
1729 // conditional load. Likewise for constants in range for LOCHI.
1730 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1731 (Subtarget->hasLoadStoreOnCond2() &&
1732 Node->getValueType(ResNo: 0).isInteger() &&
1733 Node->getValueType(ResNo: 0).getSizeInBits() <= 64 &&
1734 Op1.getOpcode() == ISD::Constant &&
1735 isInt<16>(x: cast<ConstantSDNode>(Val&: Op1)->getSExtValue()) &&
1736 !(Op0.getOpcode() == ISD::Constant &&
1737 isInt<16>(x: cast<ConstantSDNode>(Val&: Op0)->getSExtValue())))) {
1738 SDValue CCValid = Node->getOperand(Num: 2);
1739 SDValue CCMask = Node->getOperand(Num: 3);
1740 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal();
1741 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal();
1742 // Invert the condition.
1743 CCMask = CurDAG->getTargetConstant(Val: ConstCCValid ^ ConstCCMask,
1744 DL: SDLoc(Node), VT: CCMask.getValueType());
1745 SDValue Op4 = Node->getOperand(Num: 4);
1746 SDNode *UpdatedNode =
1747 CurDAG->UpdateNodeOperands(N: Node, Op1, Op2: Op0, Op3: CCValid, Op4: CCMask, Op5: Op4);
1748 if (UpdatedNode != Node) {
1749 // In case this node already exists then replace Node with it.
1750 ReplaceNode(F: Node, T: UpdatedNode);
1751 Node = UpdatedNode;
1752 }
1753 }
1754 break;
1755 }
1756
1757 case ISD::INSERT_VECTOR_ELT: {
1758 EVT VT = Node->getValueType(ResNo: 0);
1759 unsigned ElemBitSize = VT.getScalarSizeInBits();
1760 if (ElemBitSize == 32) {
1761 if (tryGather(N: Node, Opcode: SystemZ::VGEF))
1762 return;
1763 } else if (ElemBitSize == 64) {
1764 if (tryGather(N: Node, Opcode: SystemZ::VGEG))
1765 return;
1766 }
1767 break;
1768 }
1769
1770 case ISD::BUILD_VECTOR: {
1771 auto *BVN = cast<BuildVectorSDNode>(Val: Node);
1772 SystemZVectorConstantInfo VCI(BVN);
1773 if (VCI.isVectorConstantLegal(Subtarget: *Subtarget)) {
1774 loadVectorConstant(VCI, Node);
1775 return;
1776 }
1777 break;
1778 }
1779
1780 case ISD::ConstantFP: {
1781 APFloat Imm = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1782 if (Imm.isZero() || Imm.isNegZero())
1783 break;
1784 SystemZVectorConstantInfo VCI(Imm);
1785 bool Success = VCI.isVectorConstantLegal(Subtarget: *Subtarget); (void)Success;
1786 assert(Success && "Expected legal FP immediate");
1787 loadVectorConstant(VCI, Node);
1788 return;
1789 }
1790
1791 case ISD::STORE: {
1792 if (tryFoldLoadStoreIntoMemOperand(Node))
1793 return;
1794 auto *Store = cast<StoreSDNode>(Val: Node);
1795 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1796 if (ElemBitSize == 32) {
1797 if (tryScatter(Store, Opcode: SystemZ::VSCEF))
1798 return;
1799 } else if (ElemBitSize == 64) {
1800 if (tryScatter(Store, Opcode: SystemZ::VSCEG))
1801 return;
1802 }
1803 break;
1804 }
1805
1806 case ISD::ATOMIC_STORE: {
1807 auto *AtomOp = cast<AtomicSDNode>(Val: Node);
1808 // Replace the atomic_store with a regular store and select it. This is
1809 // ok since we know all store instructions <= 8 bytes are atomic, and the
1810 // 16 byte case is already handled during lowering.
1811 StoreSDNode *St = cast<StoreSDNode>(Val: CurDAG->getTruncStore(
1812 Chain: AtomOp->getChain(), dl: SDLoc(AtomOp), Val: AtomOp->getVal(),
1813 Ptr: AtomOp->getBasePtr(), SVT: AtomOp->getMemoryVT(), MMO: AtomOp->getMemOperand()));
1814 assert(St->getMemOperand()->isAtomic() && "Broken MMO.");
1815 SDNode *Chain = St;
1816 // We have to enforce sequential consistency by performing a
1817 // serialization operation after the store.
1818 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1819 Chain = CurDAG->getMachineNode(Opcode: SystemZ::Serialize, dl: SDLoc(AtomOp),
1820 VT: MVT::Other, Op1: SDValue(Chain, 0));
1821 ReplaceNode(F: Node, T: Chain);
1822 SelectCode(N: St);
1823 return;
1824 }
1825 }
1826
1827 SelectCode(N: Node);
1828}
1829
1830bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1831 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1832 std::vector<SDValue> &OutOps) {
1833 SystemZAddressingMode::AddrForm Form;
1834 SystemZAddressingMode::DispRange DispRange;
1835 SDValue Base, Disp, Index;
1836
1837 switch(ConstraintID) {
1838 default:
1839 llvm_unreachable("Unexpected asm memory constraint");
1840 case InlineAsm::ConstraintCode::i:
1841 case InlineAsm::ConstraintCode::Q:
1842 case InlineAsm::ConstraintCode::ZQ:
1843 // Accept an address with a short displacement, but no index.
1844 Form = SystemZAddressingMode::FormBD;
1845 DispRange = SystemZAddressingMode::Disp12Only;
1846 break;
1847 case InlineAsm::ConstraintCode::R:
1848 case InlineAsm::ConstraintCode::ZR:
1849 // Accept an address with a short displacement and an index.
1850 Form = SystemZAddressingMode::FormBDXNormal;
1851 DispRange = SystemZAddressingMode::Disp12Only;
1852 break;
1853 case InlineAsm::ConstraintCode::S:
1854 case InlineAsm::ConstraintCode::ZS:
1855 // Accept an address with a long displacement, but no index.
1856 Form = SystemZAddressingMode::FormBD;
1857 DispRange = SystemZAddressingMode::Disp20Only;
1858 break;
1859 case InlineAsm::ConstraintCode::T:
1860 case InlineAsm::ConstraintCode::m:
1861 case InlineAsm::ConstraintCode::o:
1862 case InlineAsm::ConstraintCode::p:
1863 case InlineAsm::ConstraintCode::ZT:
1864 // Accept an address with a long displacement and an index.
1865 // m works the same as T, as this is the most general case.
1866 // We don't really have any special handling of "offsettable"
1867 // memory addresses, so just treat o the same as m.
1868 Form = SystemZAddressingMode::FormBDXNormal;
1869 DispRange = SystemZAddressingMode::Disp20Only;
1870 break;
1871 }
1872
1873 if (selectBDXAddr(Form, DR: DispRange, Addr: Op, Base, Disp, Index)) {
1874 const TargetRegisterClass *TRC =
1875 Subtarget->getRegisterInfo()->getPointerRegClass();
1876 SDLoc DL(Base);
1877 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL, VT: MVT::i32);
1878
1879 // Make sure that the base address doesn't go into %r0.
1880 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1881 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1882 Base.getOpcode() != ISD::Register) {
1883 Base =
1884 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
1885 dl: DL, VT: Base.getValueType(),
1886 Op1: Base, Op2: RC), 0);
1887 }
1888
1889 // Make sure that the index register isn't assigned to %r0 either.
1890 if (Index.getOpcode() != ISD::Register) {
1891 Index =
1892 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
1893 dl: DL, VT: Index.getValueType(),
1894 Op1: Index, Op2: RC), 0);
1895 }
1896
1897 OutOps.push_back(x: Base);
1898 OutOps.push_back(x: Disp);
1899 OutOps.push_back(x: Index);
1900 return false;
1901 }
1902
1903 return true;
1904}
1905
1906// IsProfitableToFold - Returns true if is profitable to fold the specific
1907// operand node N of U during instruction selection that starts at Root.
1908bool
1909SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1910 SDNode *Root) const {
1911 // We want to avoid folding a LOAD into an ICMP node if as a result
1912 // we would be forced to spill the condition code into a GPR.
1913 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1914 if (!N.hasOneUse() || !U->hasOneUse())
1915 return false;
1916
1917 // The user of the CC value will usually be a CopyToReg into the
1918 // physical CC register, which in turn is glued and chained to the
1919 // actual instruction that uses the CC value. Bail out if we have
1920 // anything else than that.
1921 SDNode *CCUser = *U->user_begin();
1922 SDNode *CCRegUser = nullptr;
1923 if (CCUser->getOpcode() == ISD::CopyToReg ||
1924 cast<RegisterSDNode>(Val: CCUser->getOperand(Num: 1))->getReg() == SystemZ::CC) {
1925 for (auto *U : CCUser->users()) {
1926 if (CCRegUser == nullptr)
1927 CCRegUser = U;
1928 else if (CCRegUser != U)
1929 return false;
1930 }
1931 }
1932 if (CCRegUser == nullptr)
1933 return false;
1934
1935 // If the actual instruction is a branch, the only thing that remains to be
1936 // checked is whether the CCUser chain is a predecessor of the load.
1937 if (CCRegUser->isMachineOpcode() &&
1938 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1939 return !N->isPredecessorOf(N: CCUser->getOperand(Num: 0).getNode());
1940
1941 // Otherwise, the instruction may have multiple operands, and we need to
1942 // verify that none of them are a predecessor of the load. This is exactly
1943 // the same check that would be done by common code if the CC setter were
1944 // glued to the CC user, so simply invoke that check here.
1945 if (!IsLegalToFold(N, U, Root: CCRegUser, OptLevel, IgnoreChains: false))
1946 return false;
1947 }
1948
1949 return true;
1950}
1951
1952namespace {
1953// Represents a sequence for extracting a 0/1 value from an IPM result:
1954// (((X ^ XORValue) + AddValue) >> Bit)
1955struct IPMConversion {
1956 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1957 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1958
1959 int64_t XORValue;
1960 int64_t AddValue;
1961 unsigned Bit;
1962};
1963} // end anonymous namespace
1964
1965// Return a sequence for getting a 1 from an IPM result when CC has a
1966// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1967// The handling of CC values outside CCValid doesn't matter.
1968static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1969 // Deal with cases where the result can be taken directly from a bit
1970 // of the IPM result.
1971 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1972 return IPMConversion(0, 0, SystemZ::IPM_CC);
1973 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1974 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1975
1976 // Deal with cases where we can add a value to force the sign bit
1977 // to contain the right value. Putting the bit in 31 means we can
1978 // use SRL rather than RISBG(L), and also makes it easier to get a
1979 // 0/-1 value, so it has priority over the other tests below.
1980 //
1981 // These sequences rely on the fact that the upper two bits of the
1982 // IPM result are zero.
1983 uint64_t TopBit = uint64_t(1) << 31;
1984 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1985 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1986 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1987 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1988 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1989 | SystemZ::CCMASK_1
1990 | SystemZ::CCMASK_2)))
1991 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1992 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1993 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1994 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1995 | SystemZ::CCMASK_2
1996 | SystemZ::CCMASK_3)))
1997 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1998
1999 // Next try inverting the value and testing a bit. 0/1 could be
2000 // handled this way too, but we dealt with that case above.
2001 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
2002 return IPMConversion(-1, 0, SystemZ::IPM_CC);
2003
2004 // Handle cases where adding a value forces a non-sign bit to contain
2005 // the right value.
2006 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
2007 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
2008 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
2009 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
2010
2011 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
2012 // can be done by inverting the low CC bit and applying one of the
2013 // sign-based extractions above.
2014 if (CCMask == (CCValid & SystemZ::CCMASK_1))
2015 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
2016 if (CCMask == (CCValid & SystemZ::CCMASK_2))
2017 return IPMConversion(1 << SystemZ::IPM_CC,
2018 TopBit - (3 << SystemZ::IPM_CC), 31);
2019 if (CCMask == (CCValid & (SystemZ::CCMASK_0
2020 | SystemZ::CCMASK_1
2021 | SystemZ::CCMASK_3)))
2022 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
2023 if (CCMask == (CCValid & (SystemZ::CCMASK_0
2024 | SystemZ::CCMASK_2
2025 | SystemZ::CCMASK_3)))
2026 return IPMConversion(1 << SystemZ::IPM_CC,
2027 TopBit - (1 << SystemZ::IPM_CC), 31);
2028
2029 llvm_unreachable("Unexpected CC combination");
2030}
2031
2032SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
2033 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0));
2034 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
2035 if (!TrueOp || !FalseOp)
2036 return SDValue();
2037 if (FalseOp->getZExtValue() != 0)
2038 return SDValue();
2039 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
2040 return SDValue();
2041
2042 auto *CCValidOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 2));
2043 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 3));
2044 if (!CCValidOp || !CCMaskOp)
2045 return SDValue();
2046 int CCValid = CCValidOp->getZExtValue();
2047 int CCMask = CCMaskOp->getZExtValue();
2048
2049 SDLoc DL(Node);
2050 SDValue CCReg = Node->getOperand(Num: 4);
2051 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
2052 SDValue Result = CurDAG->getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
2053
2054 if (IPM.XORValue)
2055 Result = CurDAG->getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Result,
2056 N2: CurDAG->getConstant(Val: IPM.XORValue, DL, VT: MVT::i32));
2057
2058 if (IPM.AddValue)
2059 Result =
2060 CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: Result,
2061 N2: CurDAG->getSignedConstant(Val: IPM.AddValue, DL, VT: MVT::i32));
2062
2063 EVT VT = Node->getValueType(ResNo: 0);
2064 if (VT == MVT::i32 && IPM.Bit == 31) {
2065 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
2066 Result = CurDAG->getNode(Opcode: ShiftOp, DL, VT: MVT::i32, N1: Result,
2067 N2: CurDAG->getConstant(Val: IPM.Bit, DL, VT: MVT::i32));
2068 } else {
2069 if (VT != MVT::i32)
2070 Result = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Result);
2071
2072 if (TrueOp->getSExtValue() == 1) {
2073 // The SHR/AND sequence should get optimized to an RISBG.
2074 Result = CurDAG->getNode(Opcode: ISD::SRL, DL, VT, N1: Result,
2075 N2: CurDAG->getConstant(Val: IPM.Bit, DL, VT: MVT::i32));
2076 Result = CurDAG->getNode(Opcode: ISD::AND, DL, VT, N1: Result,
2077 N2: CurDAG->getConstant(Val: 1, DL, VT));
2078 } else {
2079 // Sign-extend from IPM.Bit using a pair of shifts.
2080 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
2081 int SraAmt = VT.getSizeInBits() - 1;
2082 Result = CurDAG->getNode(Opcode: ISD::SHL, DL, VT, N1: Result,
2083 N2: CurDAG->getConstant(Val: ShlAmt, DL, VT: MVT::i32));
2084 Result = CurDAG->getNode(Opcode: ISD::SRA, DL, VT, N1: Result,
2085 N2: CurDAG->getConstant(Val: SraAmt, DL, VT: MVT::i32));
2086 }
2087 }
2088
2089 return Result;
2090}
2091
2092bool SystemZDAGToDAGISel::shouldSelectForReassoc(SDNode *N) const {
2093 EVT VT = N->getValueType(ResNo: 0);
2094 assert(VT.isFloatingPoint() && "Expected FP SDNode");
2095 return N->getFlags().hasAllowReassociation() &&
2096 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&
2097 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&
2098 !N->isStrictFPOpcode();
2099}
2100
2101void SystemZDAGToDAGISel::PreprocessISelDAG() {
2102 // If we have conditional immediate loads, we always prefer
2103 // using those over an IPM sequence.
2104 if (Subtarget->hasLoadStoreOnCond2())
2105 return;
2106
2107 bool MadeChange = false;
2108
2109 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
2110 E = CurDAG->allnodes_end();
2111 I != E;) {
2112 SDNode *N = &*I++;
2113 if (N->use_empty())
2114 continue;
2115
2116 SDValue Res;
2117 switch (N->getOpcode()) {
2118 default: break;
2119 case SystemZISD::SELECT_CCMASK:
2120 Res = expandSelectBoolean(Node: N);
2121 break;
2122 }
2123
2124 if (Res) {
2125 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
2126 LLVM_DEBUG(N->dump(CurDAG));
2127 LLVM_DEBUG(dbgs() << "\nNew: ");
2128 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
2129 LLVM_DEBUG(dbgs() << "\n");
2130
2131 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
2132 MadeChange = true;
2133 }
2134 }
2135
2136 if (MadeChange)
2137 CurDAG->RemoveDeadNodes();
2138}
2139