1//===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the PowerPC implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCInstrInfo.h"
14#include "MCTargetDesc/PPCPredicates.h"
15#include "PPC.h"
16#include "PPCHazardRecognizers.h"
17#include "PPCInstrBuilder.h"
18#include "PPCMachineFunctionInfo.h"
19#include "PPCTargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/LiveIntervals.h"
23#include "llvm/CodeGen/LivePhysRegs.h"
24#include "llvm/CodeGen/MachineCombinerPattern.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/PseudoSourceValue.h"
31#include "llvm/CodeGen/RegisterClassInfo.h"
32#include "llvm/CodeGen/RegisterPressure.h"
33#include "llvm/CodeGen/RegisterScavenging.h"
34#include "llvm/CodeGen/ScheduleDAG.h"
35#include "llvm/CodeGen/SlotIndexes.h"
36#include "llvm/CodeGen/StackMaps.h"
37#include "llvm/IR/Module.h"
38#include "llvm/MC/MCInst.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/raw_ostream.h"
44
45using namespace llvm;
46
47#define DEBUG_TYPE "ppc-instr-info"
48
49#define GET_INSTRMAP_INFO
50#define GET_INSTRINFO_CTOR_DTOR
51#include "PPCGenInstrInfo.inc"
52
53STATISTIC(NumStoreSPILLVSRRCAsVec,
54 "Number of spillvsrrc spilled to stack as vec");
55STATISTIC(NumStoreSPILLVSRRCAsGpr,
56 "Number of spillvsrrc spilled to stack as gpr");
57STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
58STATISTIC(CmpIselsConverted,
59 "Number of ISELs that depend on comparison of constants converted");
60STATISTIC(MissedConvertibleImmediateInstrs,
61 "Number of compare-immediate instructions fed by constants");
62STATISTIC(NumRcRotatesConvertedToRcAnd,
63 "Number of record-form rotates converted to record-form andi");
64
65static cl::
66opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
67 cl::desc("Disable analysis for CTR loops"));
68
69static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
70cl::desc("Disable compare instruction optimization"), cl::Hidden);
71
72static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
73cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
74cl::Hidden);
75
76static cl::opt<bool>
77UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
78 cl::desc("Use the old (incorrect) instruction latency calculation"));
79
80static cl::opt<float>
81 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(Val: 1.5),
82 cl::desc("register pressure factor for the transformations."));
83
84static cl::opt<bool> EnableFMARegPressureReduction(
85 "ppc-fma-rp-reduction", cl::Hidden, cl::init(Val: true),
86 cl::desc("enable register pressure reduce in machine combiner pass."));
87
88// Pin the vtable to this file.
89void PPCInstrInfo::anchor() {}
90
91PPCInstrInfo::PPCInstrInfo(const PPCSubtarget &STI)
92 : PPCGenInstrInfo(STI, RI, PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
93 /* CatchRetOpcode */ -1,
94 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
95 Subtarget(STI), RI(STI.getTargetMachine()) {}
96
97/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
98/// this target when scheduling the DAG.
99ScheduleHazardRecognizer *
100PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
101 const ScheduleDAG *DAG) const {
102 unsigned Directive =
103 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
104 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
105 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
106 const InstrItineraryData *II =
107 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
108 return new ScoreboardHazardRecognizer(II, DAG);
109 }
110
111 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
112}
113
114/// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
115/// to use for this target when scheduling the DAG.
116ScheduleHazardRecognizer *
117PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
118 const ScheduleDAG *DAG) const {
119 unsigned Directive =
120 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
121
122 // FIXME: Leaving this as-is until we have POWER9 scheduling info
123 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
124 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
125
126 // Most subtargets use a PPC970 recognizer.
127 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
128 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
129 assert(DAG->TII && "No InstrInfo?");
130
131 return new PPCHazardRecognizer970(*DAG);
132 }
133
134 return new ScoreboardHazardRecognizer(II, DAG);
135}
136
137unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
138 const MachineInstr &MI,
139 unsigned *PredCost) const {
140 if (!ItinData || UseOldLatencyCalc)
141 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
142
143 // The default implementation of getInstrLatency calls getStageLatency, but
144 // getStageLatency does not do the right thing for us. While we have
145 // itinerary, most cores are fully pipelined, and so the itineraries only
146 // express the first part of the pipeline, not every stage. Instead, we need
147 // to use the listed output operand cycle number (using operand 0 here, which
148 // is an output).
149
150 unsigned Latency = 1;
151 unsigned DefClass = MI.getDesc().getSchedClass();
152 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
153 const MachineOperand &MO = MI.getOperand(i);
154 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
155 continue;
156
157 std::optional<unsigned> Cycle = ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: i);
158 if (!Cycle)
159 continue;
160
161 Latency = std::max(a: Latency, b: *Cycle);
162 }
163
164 return Latency;
165}
166
167std::optional<unsigned> PPCInstrInfo::getOperandLatency(
168 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
169 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
170 std::optional<unsigned> Latency = PPCGenInstrInfo::getOperandLatency(
171 ItinData, DefMI, DefIdx, UseMI, UseIdx);
172
173 if (!DefMI.getParent())
174 return Latency;
175
176 const MachineOperand &DefMO = DefMI.getOperand(i: DefIdx);
177 Register Reg = DefMO.getReg();
178
179 bool IsRegCR;
180 if (Reg.isVirtual()) {
181 const MachineRegisterInfo *MRI =
182 &DefMI.getParent()->getParent()->getRegInfo();
183 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(RC: &PPC::CRRCRegClass) ||
184 MRI->getRegClass(Reg)->hasSuperClassEq(RC: &PPC::CRBITRCRegClass);
185 } else {
186 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
187 PPC::CRBITRCRegClass.contains(Reg);
188 }
189
190 if (UseMI.isBranch() && IsRegCR) {
191 if (!Latency)
192 Latency = getInstrLatency(ItinData, MI: DefMI);
193
194 // On some cores, there is an additional delay between writing to a condition
195 // register, and using it from a branch.
196 unsigned Directive = Subtarget.getCPUDirective();
197 switch (Directive) {
198 default: break;
199 case PPC::DIR_7400:
200 case PPC::DIR_750:
201 case PPC::DIR_970:
202 case PPC::DIR_E5500:
203 case PPC::DIR_PWR4:
204 case PPC::DIR_PWR5:
205 case PPC::DIR_PWR5X:
206 case PPC::DIR_PWR6:
207 case PPC::DIR_PWR6X:
208 case PPC::DIR_PWR7:
209 case PPC::DIR_PWR8:
210 // FIXME: Is this needed for POWER9?
211 Latency = *Latency + 2;
212 break;
213 }
214 }
215
216 return Latency;
217}
218
219void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
220 uint32_t Flags) const {
221 MI.setFlags(Flags);
222 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
223 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
224 MI.clearFlag(Flag: MachineInstr::MIFlag::IsExact);
225}
226
227// This function does not list all associative and commutative operations, but
228// only those worth feeding through the machine combiner in an attempt to
229// reduce the critical path. Mostly, this means floating-point operations,
230// because they have high latencies(>=5) (compared to other operations, such as
231// and/or, which are also associative and commutative, but have low latencies).
232bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
233 bool Invert) const {
234 if (Invert)
235 return false;
236 switch (Inst.getOpcode()) {
237 // Floating point:
238 // FP Add:
239 case PPC::FADD:
240 case PPC::FADDS:
241 // FP Multiply:
242 case PPC::FMUL:
243 case PPC::FMULS:
244 // Altivec Add:
245 case PPC::VADDFP:
246 // VSX Add:
247 case PPC::XSADDDP:
248 case PPC::XVADDDP:
249 case PPC::XVADDSP:
250 case PPC::XSADDSP:
251 // VSX Multiply:
252 case PPC::XSMULDP:
253 case PPC::XVMULDP:
254 case PPC::XVMULSP:
255 case PPC::XSMULSP:
256 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
257 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
258 // Fixed point:
259 // Multiply:
260 case PPC::MULHD:
261 case PPC::MULLD:
262 case PPC::MULHW:
263 case PPC::MULLW:
264 return true;
265 default:
266 return false;
267 }
268}
269
270#define InfoArrayIdxFMAInst 0
271#define InfoArrayIdxFAddInst 1
272#define InfoArrayIdxFMULInst 2
273#define InfoArrayIdxAddOpIdx 3
274#define InfoArrayIdxMULOpIdx 4
275#define InfoArrayIdxFSubInst 5
276// Array keeps info for FMA instructions:
277// Index 0(InfoArrayIdxFMAInst): FMA instruction;
278// Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
279// Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
280// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
281// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
282// second MUL operand index is plus 1;
283// Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
284static const uint16_t FMAOpIdxInfo[][6] = {
285 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
286 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
287 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
288 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
289 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
290 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
291 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
292
293// Check if an opcode is a FMA instruction. If it is, return the index in array
294// FMAOpIdxInfo. Otherwise, return -1.
295int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
296 for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
297 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
298 return I;
299 return -1;
300}
301
302// On PowerPC target, we have two kinds of patterns related to FMA:
303// 1: Improve ILP.
304// Try to reassociate FMA chains like below:
305//
306// Pattern 1:
307// A = FADD X, Y (Leaf)
308// B = FMA A, M21, M22 (Prev)
309// C = FMA B, M31, M32 (Root)
310// -->
311// A = FMA X, M21, M22
312// B = FMA Y, M31, M32
313// C = FADD A, B
314//
315// Pattern 2:
316// A = FMA X, M11, M12 (Leaf)
317// B = FMA A, M21, M22 (Prev)
318// C = FMA B, M31, M32 (Root)
319// -->
320// A = FMUL M11, M12
321// B = FMA X, M21, M22
322// D = FMA A, M31, M32
323// C = FADD B, D
324//
325// breaking the dependency between A and B, allowing FMA to be executed in
326// parallel (or back-to-back in a pipeline) instead of depending on each other.
327//
328// 2: Reduce register pressure.
329// Try to reassociate FMA with FSUB and a constant like below:
330// C is a floating point const.
331//
332// Pattern 1:
333// A = FSUB X, Y (Leaf)
334// D = FMA B, C, A (Root)
335// -->
336// A = FMA B, Y, -C
337// D = FMA A, X, C
338//
339// Pattern 2:
340// A = FSUB X, Y (Leaf)
341// D = FMA B, A, C (Root)
342// -->
343// A = FMA B, Y, -C
344// D = FMA A, X, C
345//
346// Before the transformation, A must be assigned with different hardware
347// register with D. After the transformation, A and D must be assigned with
348// same hardware register due to TIE attribute of FMA instructions.
349//
350bool PPCInstrInfo::getFMAPatterns(MachineInstr &Root,
351 SmallVectorImpl<unsigned> &Patterns,
352 bool DoRegPressureReduce) const {
353 MachineBasicBlock *MBB = Root.getParent();
354 const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
355
356 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
357 for (const auto &MO : Instr.explicit_operands())
358 if (!(MO.isReg() && MO.getReg().isVirtual()))
359 return false;
360 return true;
361 };
362
363 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
364 unsigned OpType) {
365 if (Instr.getOpcode() !=
366 FMAOpIdxInfo[getFMAOpIdxInfo(Opcode: Root.getOpcode())][OpType])
367 return false;
368
369 // Instruction can be reassociated.
370 // fast math flags may prohibit reassociation.
371 if (!(Instr.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
372 Instr.getFlag(Flag: MachineInstr::MIFlag::FmNsz)))
373 return false;
374
375 // Instruction operands are virtual registers for reassociation.
376 if (!IsAllOpsVirtualReg(Instr))
377 return false;
378
379 // For register pressure reassociation, the FSub must have only one use as
380 // we want to delete the sub to save its def.
381 if (OpType == InfoArrayIdxFSubInst &&
382 !MRI->hasOneNonDBGUse(RegNo: Instr.getOperand(i: 0).getReg()))
383 return false;
384
385 return true;
386 };
387
388 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
389 int16_t &MulOpIdx, bool IsLeaf) {
390 int16_t Idx = getFMAOpIdxInfo(Opcode: Instr.getOpcode());
391 if (Idx < 0)
392 return false;
393
394 // Instruction can be reassociated.
395 // fast math flags may prohibit reassociation.
396 if (!(Instr.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
397 Instr.getFlag(Flag: MachineInstr::MIFlag::FmNsz)))
398 return false;
399
400 // Instruction operands are virtual registers for reassociation.
401 if (!IsAllOpsVirtualReg(Instr))
402 return false;
403
404 MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
405 if (IsLeaf)
406 return true;
407
408 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
409
410 const MachineOperand &OpAdd = Instr.getOperand(i: AddOpIdx);
411 MachineInstr *MIAdd = MRI->getUniqueVRegDef(Reg: OpAdd.getReg());
412 // If 'add' operand's def is not in current block, don't do ILP related opt.
413 if (!MIAdd || MIAdd->getParent() != MBB)
414 return false;
415
416 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
417 // as this fma will be changed later.
418 return MRI->hasOneNonDBGUse(RegNo: OpAdd.getReg());
419 };
420
421 int16_t AddOpIdx = -1;
422 int16_t MulOpIdx = -1;
423
424 bool IsUsedOnceL = false;
425 bool IsUsedOnceR = false;
426 MachineInstr *MULInstrL = nullptr;
427 MachineInstr *MULInstrR = nullptr;
428
429 auto IsRPReductionCandidate = [&]() {
430 // Currently, we only support float and double.
431 // FIXME: add support for other types.
432 unsigned Opcode = Root.getOpcode();
433 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
434 return false;
435
436 // Root must be a valid FMA like instruction.
437 // Treat it as leaf as we don't care its add operand.
438 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
439 assert((MulOpIdx >= 0) && "mul operand index not right!");
440 Register MULRegL = RI.lookThruSingleUseCopyChain(
441 SrcReg: Root.getOperand(i: MulOpIdx).getReg(), MRI);
442 Register MULRegR = RI.lookThruSingleUseCopyChain(
443 SrcReg: Root.getOperand(i: MulOpIdx + 1).getReg(), MRI);
444 if (!MULRegL && !MULRegR)
445 return false;
446
447 if (MULRegL && !MULRegR) {
448 MULRegR =
449 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: MulOpIdx + 1).getReg(), MRI);
450 IsUsedOnceL = true;
451 } else if (!MULRegL && MULRegR) {
452 MULRegL = RI.lookThruCopyLike(SrcReg: Root.getOperand(i: MulOpIdx).getReg(), MRI);
453 IsUsedOnceR = true;
454 } else {
455 IsUsedOnceL = true;
456 IsUsedOnceR = true;
457 }
458
459 if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
460 return false;
461
462 MULInstrL = MRI->getVRegDef(Reg: MULRegL);
463 MULInstrR = MRI->getVRegDef(Reg: MULRegR);
464 return MULInstrL && MULInstrR;
465 }
466 return false;
467 };
468
469 // Register pressure fma reassociation patterns.
470 if (DoRegPressureReduce && IsRPReductionCandidate()) {
471 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
472 // Register pressure pattern 1
473 if (isLoadFromConstantPool(I: MULInstrL) && IsUsedOnceR &&
474 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
475 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
476 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_BCA);
477 return true;
478 }
479
480 // Register pressure pattern 2
481 if ((isLoadFromConstantPool(I: MULInstrR) && IsUsedOnceL &&
482 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
483 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
484 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_BAC);
485 return true;
486 }
487 }
488
489 // ILP fma reassociation patterns.
490 // Root must be a valid FMA like instruction.
491 AddOpIdx = -1;
492 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
493 return false;
494
495 assert((AddOpIdx >= 0) && "add operand index not right!");
496
497 Register RegB = Root.getOperand(i: AddOpIdx).getReg();
498 MachineInstr *Prev = MRI->getUniqueVRegDef(Reg: RegB);
499
500 // Prev must be a valid FMA like instruction.
501 AddOpIdx = -1;
502 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
503 return false;
504
505 assert((AddOpIdx >= 0) && "add operand index not right!");
506
507 Register RegA = Prev->getOperand(i: AddOpIdx).getReg();
508 MachineInstr *Leaf = MRI->getUniqueVRegDef(Reg: RegA);
509 AddOpIdx = -1;
510 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
511 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
512 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
513 return true;
514 }
515 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
516 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM);
517 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
518 return true;
519 }
520 return false;
521}
522
523void PPCInstrInfo::finalizeInsInstrs(
524 MachineInstr &Root, unsigned &Pattern,
525 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
526 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
527
528 MachineFunction *MF = Root.getMF();
529 MachineRegisterInfo *MRI = &MF->getRegInfo();
530 MachineConstantPool *MCP = MF->getConstantPool();
531
532 int16_t Idx = getFMAOpIdxInfo(Opcode: Root.getOpcode());
533 if (Idx < 0)
534 return;
535
536 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
537
538 // For now we only need to fix up placeholder for register pressure reduce
539 // patterns.
540 Register ConstReg = 0;
541 switch (Pattern) {
542 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
543 ConstReg =
544 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx).getReg(), MRI);
545 break;
546 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
547 ConstReg =
548 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx + 1).getReg(), MRI);
549 break;
550 default:
551 // Not register pressure reduce patterns.
552 return;
553 }
554
555 MachineInstr *ConstDefInstr = MRI->getVRegDef(Reg: ConstReg);
556 // Get const value from const pool.
557 const Constant *C = getConstantFromConstantPool(I: ConstDefInstr);
558 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
559
560 // Get negative fp const.
561 APFloat F1((dyn_cast<ConstantFP>(Val: C))->getValueAPF());
562 F1.changeSign();
563 Constant *NegC = ConstantFP::get(Context&: dyn_cast<ConstantFP>(Val: C)->getContext(), V: F1);
564 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: C->getType());
565
566 // Put negative fp const into constant pool.
567 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(C: NegC, Alignment);
568
569 MachineOperand *Placeholder = nullptr;
570 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
571 for (auto *Inst : InsInstrs) {
572 for (MachineOperand &Operand : Inst->explicit_operands()) {
573 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
574 if (Operand.getReg() == PPC::ZERO8) {
575 Placeholder = &Operand;
576 break;
577 }
578 }
579 }
580
581 assert(Placeholder && "Placeholder does not exist!");
582
583 // Generate instructions to load the const fp from constant pool.
584 // We only support PPC64 and medium code model.
585 Register LoadNewConst =
586 generateLoadForNewConst(Idx: ConstPoolIdx, MI: &Root, Ty: C->getType(), InsInstrs);
587
588 // Fill the placeholder with the new load from constant pool.
589 Placeholder->setReg(LoadNewConst);
590}
591
592bool PPCInstrInfo::shouldReduceRegisterPressure(
593 const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
594
595 if (!EnableFMARegPressureReduction)
596 return false;
597
598 // Currently, we only enable register pressure reducing in machine combiner
599 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
600 // support.
601 //
602 // So we need following instructions to access a TOC entry:
603 //
604 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
605 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
606 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
607 //
608 // FIXME: add more supported targets, like Small and Large code model, PPC32,
609 // AIX.
610 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
611 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
612 return false;
613
614 const MachineFunction *MF = MBB->getParent();
615 const MachineRegisterInfo *MRI = &MF->getRegInfo();
616
617 auto GetMBBPressure =
618 [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
619 RegionPressure Pressure;
620 RegPressureTracker RPTracker(Pressure);
621
622 // Initialize the register pressure tracker.
623 RPTracker.init(mf: MBB->getParent(), rci: RegClassInfo, lis: nullptr, mbb: MBB, pos: MBB->end(),
624 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
625
626 for (const auto &MI : reverse(C: *MBB)) {
627 if (MI.isDebugValue() || MI.isDebugLabel())
628 continue;
629 RegisterOperands RegOpers;
630 RegOpers.collect(MI, TRI: RI, MRI: *MRI, TrackLaneMasks: false, IgnoreDead: false);
631 RPTracker.recedeSkipDebugValues();
632 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
633 RPTracker.recede(RegOpers);
634 }
635
636 // Close the RPTracker to finalize live ins.
637 RPTracker.closeRegion();
638
639 return RPTracker.getPressure().MaxSetPressure;
640 };
641
642 // For now we only care about float and double type fma.
643 unsigned VSSRCLimit =
644 RegClassInfo->getRegPressureSetLimit(Idx: PPC::RegisterPressureSets::VSSRC);
645
646 // Only reduce register pressure when pressure is high.
647 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
648 (float)VSSRCLimit * FMARPFactor;
649}
650
651bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
652 // I has only one memory operand which is load from constant pool.
653 if (!I->hasOneMemOperand())
654 return false;
655
656 MachineMemOperand *Op = I->memoperands()[0];
657 return Op->isLoad() && Op->getPseudoValue() &&
658 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
659}
660
661Register PPCInstrInfo::generateLoadForNewConst(
662 unsigned Idx, MachineInstr *MI, Type *Ty,
663 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
664 // Now we only support PPC64, Medium code model and P9 with vector.
665 // We have immutable pattern to access const pool. See function
666 // shouldReduceRegisterPressure.
667 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
668 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
669 "Target not supported!\n");
670
671 MachineFunction *MF = MI->getMF();
672 MachineRegisterInfo *MRI = &MF->getRegInfo();
673
674 // Generate ADDIStocHA8
675 Register VReg1 = MRI->createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
676 MachineInstrBuilder TOCOffset =
677 BuildMI(MF&: *MF, MIMD: MI->getDebugLoc(), MCID: get(Opcode: PPC::ADDIStocHA8), DestReg: VReg1)
678 .addReg(RegNo: PPC::X2)
679 .addConstantPoolIndex(Idx);
680
681 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
682 "Only float and double are supported!");
683
684 unsigned LoadOpcode;
685 // Should be float type or double type.
686 if (Ty->isFloatTy())
687 LoadOpcode = PPC::DFLOADf32;
688 else
689 LoadOpcode = PPC::DFLOADf64;
690
691 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MI->getOperand(i: 0).getReg());
692 Register VReg2 = MRI->createVirtualRegister(RegClass: RC);
693 MachineMemOperand *MMO = MF->getMachineMemOperand(
694 PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF), F: MachineMemOperand::MOLoad,
695 Size: Ty->getScalarSizeInBits() / 8, BaseAlignment: MF->getDataLayout().getPrefTypeAlign(Ty));
696
697 // Generate Load from constant pool.
698 MachineInstrBuilder Load =
699 BuildMI(MF&: *MF, MIMD: MI->getDebugLoc(), MCID: get(Opcode: LoadOpcode), DestReg: VReg2)
700 .addConstantPoolIndex(Idx)
701 .addReg(RegNo: VReg1, Flags: getKillRegState(B: true))
702 .addMemOperand(MMO);
703
704 Load->getOperand(i: 1).setTargetFlags(PPCII::MO_TOC_LO);
705
706 // Insert the toc load instructions into InsInstrs.
707 InsInstrs.insert(I: InsInstrs.begin(), Elt: Load);
708 InsInstrs.insert(I: InsInstrs.begin(), Elt: TOCOffset);
709 return VReg2;
710}
711
712// This function returns the const value in constant pool if the \p I is a load
713// from constant pool.
714const Constant *
715PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
716 MachineFunction *MF = I->getMF();
717 MachineRegisterInfo *MRI = &MF->getRegInfo();
718 MachineConstantPool *MCP = MF->getConstantPool();
719 assert(I->mayLoad() && "Should be a load instruction.\n");
720 for (auto MO : I->uses()) {
721 if (!MO.isReg())
722 continue;
723 Register Reg = MO.getReg();
724 if (Reg == 0 || !Reg.isVirtual())
725 continue;
726 // Find the toc address.
727 MachineInstr *DefMI = MRI->getVRegDef(Reg);
728 for (auto MO2 : DefMI->uses())
729 if (MO2.isCPI())
730 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
731 }
732 return nullptr;
733}
734
735CombinerObjective PPCInstrInfo::getCombinerObjective(unsigned Pattern) const {
736 switch (Pattern) {
737 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
738 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
739 return CombinerObjective::MustReduceDepth;
740 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
741 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
742 return CombinerObjective::MustReduceRegisterPressure;
743 default:
744 return TargetInstrInfo::getCombinerObjective(Pattern);
745 }
746}
747
748bool PPCInstrInfo::getMachineCombinerPatterns(
749 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
750 bool DoRegPressureReduce) const {
751 // Using the machine combiner in this way is potentially expensive, so
752 // restrict to when aggressive optimizations are desired.
753 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOptLevel::Aggressive)
754 return false;
755
756 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
757 return true;
758
759 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
760 DoRegPressureReduce);
761}
762
763void PPCInstrInfo::genAlternativeCodeSequence(
764 MachineInstr &Root, unsigned Pattern,
765 SmallVectorImpl<MachineInstr *> &InsInstrs,
766 SmallVectorImpl<MachineInstr *> &DelInstrs,
767 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
768 switch (Pattern) {
769 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
770 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
771 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
772 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
773 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
774 break;
775 default:
776 // Reassociate default patterns.
777 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
778 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
779 break;
780 }
781}
782
783void PPCInstrInfo::reassociateFMA(
784 MachineInstr &Root, unsigned Pattern,
785 SmallVectorImpl<MachineInstr *> &InsInstrs,
786 SmallVectorImpl<MachineInstr *> &DelInstrs,
787 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
788 MachineFunction *MF = Root.getMF();
789 MachineRegisterInfo &MRI = MF->getRegInfo();
790 MachineOperand &OpC = Root.getOperand(i: 0);
791 Register RegC = OpC.getReg();
792 const TargetRegisterClass *RC = MRI.getRegClass(Reg: RegC);
793 MRI.constrainRegClass(Reg: RegC, RC);
794
795 unsigned FmaOp = Root.getOpcode();
796 int16_t Idx = getFMAOpIdxInfo(Opcode: FmaOp);
797 assert(Idx >= 0 && "Root must be a FMA instruction");
798
799 bool IsILPReassociate =
800 (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
801 (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
802
803 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
804 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
805
806 MachineInstr *Prev = nullptr;
807 MachineInstr *Leaf = nullptr;
808 switch (Pattern) {
809 default:
810 llvm_unreachable("not recognized pattern!");
811 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
812 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
813 Prev = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: AddOpIdx).getReg());
814 Leaf = MRI.getUniqueVRegDef(Reg: Prev->getOperand(i: AddOpIdx).getReg());
815 break;
816 case PPCMachineCombinerPattern::REASSOC_XY_BAC: {
817 Register MULReg =
818 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx).getReg(), MRI: &MRI);
819 Leaf = MRI.getVRegDef(Reg: MULReg);
820 break;
821 }
822 case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
823 Register MULReg =
824 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx + 1).getReg(), MRI: &MRI);
825 Leaf = MRI.getVRegDef(Reg: MULReg);
826 break;
827 }
828 }
829
830 uint32_t IntersectedFlags = 0;
831 if (IsILPReassociate)
832 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
833 else
834 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
835
836 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
837 bool &KillFlag) {
838 Reg = Operand.getReg();
839 MRI.constrainRegClass(Reg, RC);
840 KillFlag = Operand.isKill();
841 };
842
843 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
844 Register &MulOp2, Register &AddOp,
845 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
846 bool &AddOpKillFlag) {
847 GetOperandInfo(Instr.getOperand(i: FirstMulOpIdx), MulOp1, MulOp1KillFlag);
848 GetOperandInfo(Instr.getOperand(i: FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
849 GetOperandInfo(Instr.getOperand(i: AddOpIdx), AddOp, AddOpKillFlag);
850 };
851
852 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
853 RegA21, RegB;
854 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
855 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
856 KillA11 = false, KillA21 = false, KillB = false;
857
858 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
859
860 if (IsILPReassociate)
861 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
862
863 if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
864 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
865 GetOperandInfo(Leaf->getOperand(i: AddOpIdx), RegX, KillX);
866 } else if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) {
867 GetOperandInfo(Leaf->getOperand(i: 1), RegX, KillX);
868 GetOperandInfo(Leaf->getOperand(i: 2), RegY, KillY);
869 } else {
870 // Get FSUB instruction info.
871 GetOperandInfo(Leaf->getOperand(i: 1), RegX, KillX);
872 GetOperandInfo(Leaf->getOperand(i: 2), RegY, KillY);
873 }
874
875 // Create new virtual registers for the new results instead of
876 // recycling legacy ones because the MachineCombiner's computation of the
877 // critical path requires a new register definition rather than an existing
878 // one.
879 // For register pressure reassociation, we only need create one virtual
880 // register for the new fma.
881 Register NewVRA = MRI.createVirtualRegister(RegClass: RC);
882 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRA, y: 0));
883
884 Register NewVRB = 0;
885 if (IsILPReassociate) {
886 NewVRB = MRI.createVirtualRegister(RegClass: RC);
887 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRB, y: 1));
888 }
889
890 Register NewVRD = 0;
891 if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
892 NewVRD = MRI.createVirtualRegister(RegClass: RC);
893 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRD, y: 2));
894 }
895
896 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
897 Register RegMul1, bool KillRegMul1,
898 Register RegMul2, bool KillRegMul2) {
899 MI->getOperand(i: AddOpIdx).setReg(RegAdd);
900 MI->getOperand(i: AddOpIdx).setIsKill(KillAdd);
901 MI->getOperand(i: FirstMulOpIdx).setReg(RegMul1);
902 MI->getOperand(i: FirstMulOpIdx).setIsKill(KillRegMul1);
903 MI->getOperand(i: FirstMulOpIdx + 1).setReg(RegMul2);
904 MI->getOperand(i: FirstMulOpIdx + 1).setIsKill(KillRegMul2);
905 };
906
907 MachineInstrBuilder NewARegPressure, NewCRegPressure;
908 switch (Pattern) {
909 default:
910 llvm_unreachable("not recognized pattern!");
911 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM: {
912 // Create new instructions for insertion.
913 MachineInstrBuilder MINewB =
914 BuildMI(MF&: *MF, MIMD: Prev->getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRB)
915 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
916 .addReg(RegNo: RegM21, Flags: getKillRegState(B: KillM21))
917 .addReg(RegNo: RegM22, Flags: getKillRegState(B: KillM22));
918 MachineInstrBuilder MINewA =
919 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRA)
920 .addReg(RegNo: RegY, Flags: getKillRegState(B: KillY))
921 .addReg(RegNo: RegM31, Flags: getKillRegState(B: KillM31))
922 .addReg(RegNo: RegM32, Flags: getKillRegState(B: KillM32));
923 // If AddOpIdx is not 1, adjust the order.
924 if (AddOpIdx != 1) {
925 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
926 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
927 }
928
929 MachineInstrBuilder MINewC =
930 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(),
931 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), DestReg: RegC)
932 .addReg(RegNo: NewVRB, Flags: getKillRegState(B: true))
933 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true));
934
935 // Update flags for newly created instructions.
936 setSpecialOperandAttr(MI&: *MINewA, Flags: IntersectedFlags);
937 setSpecialOperandAttr(MI&: *MINewB, Flags: IntersectedFlags);
938 setSpecialOperandAttr(MI&: *MINewC, Flags: IntersectedFlags);
939
940 // Record new instructions for insertion.
941 InsInstrs.push_back(Elt: MINewA);
942 InsInstrs.push_back(Elt: MINewB);
943 InsInstrs.push_back(Elt: MINewC);
944 break;
945 }
946 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
947 assert(NewVRD && "new FMA register not created!");
948 // Create new instructions for insertion.
949 MachineInstrBuilder MINewA =
950 BuildMI(MF&: *MF, MIMD: Leaf->getDebugLoc(),
951 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), DestReg: NewVRA)
952 .addReg(RegNo: RegM11, Flags: getKillRegState(B: KillM11))
953 .addReg(RegNo: RegM12, Flags: getKillRegState(B: KillM12));
954 MachineInstrBuilder MINewB =
955 BuildMI(MF&: *MF, MIMD: Prev->getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRB)
956 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
957 .addReg(RegNo: RegM21, Flags: getKillRegState(B: KillM21))
958 .addReg(RegNo: RegM22, Flags: getKillRegState(B: KillM22));
959 MachineInstrBuilder MINewD =
960 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRD)
961 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true))
962 .addReg(RegNo: RegM31, Flags: getKillRegState(B: KillM31))
963 .addReg(RegNo: RegM32, Flags: getKillRegState(B: KillM32));
964 // If AddOpIdx is not 1, adjust the order.
965 if (AddOpIdx != 1) {
966 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
967 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
968 KillM32);
969 }
970
971 MachineInstrBuilder MINewC =
972 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(),
973 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), DestReg: RegC)
974 .addReg(RegNo: NewVRB, Flags: getKillRegState(B: true))
975 .addReg(RegNo: NewVRD, Flags: getKillRegState(B: true));
976
977 // Update flags for newly created instructions.
978 setSpecialOperandAttr(MI&: *MINewA, Flags: IntersectedFlags);
979 setSpecialOperandAttr(MI&: *MINewB, Flags: IntersectedFlags);
980 setSpecialOperandAttr(MI&: *MINewD, Flags: IntersectedFlags);
981 setSpecialOperandAttr(MI&: *MINewC, Flags: IntersectedFlags);
982
983 // Record new instructions for insertion.
984 InsInstrs.push_back(Elt: MINewA);
985 InsInstrs.push_back(Elt: MINewB);
986 InsInstrs.push_back(Elt: MINewD);
987 InsInstrs.push_back(Elt: MINewC);
988 break;
989 }
990 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
991 case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
992 Register VarReg;
993 bool KillVarReg = false;
994 if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_BCA) {
995 VarReg = RegM31;
996 KillVarReg = KillM31;
997 } else {
998 VarReg = RegM32;
999 KillVarReg = KillM32;
1000 }
1001 // We don't want to get negative const from memory pool too early, as the
1002 // created entry will not be deleted even if it has no users. Since all
1003 // operand of Leaf and Root are virtual register, we use zero register
1004 // here as a placeholder. When the InsInstrs is selected in
1005 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1006 // with a virtual register which is a load from constant pool.
1007 NewARegPressure = BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRA)
1008 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegB))
1009 .addReg(RegNo: RegY, Flags: getKillRegState(B: KillY))
1010 .addReg(RegNo: PPC::ZERO8);
1011 NewCRegPressure = BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: RegC)
1012 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true))
1013 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
1014 .addReg(RegNo: VarReg, Flags: getKillRegState(B: KillVarReg));
1015 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1016 // both at index 1, no need to adjust.
1017 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1018 // the operand index here.
1019 break;
1020 }
1021 }
1022
1023 if (!IsILPReassociate) {
1024 setSpecialOperandAttr(MI&: *NewARegPressure, Flags: IntersectedFlags);
1025 setSpecialOperandAttr(MI&: *NewCRegPressure, Flags: IntersectedFlags);
1026
1027 InsInstrs.push_back(Elt: NewARegPressure);
1028 InsInstrs.push_back(Elt: NewCRegPressure);
1029 }
1030
1031 assert(!InsInstrs.empty() &&
1032 "Insertion instructions set should not be empty!");
1033
1034 // Record old instructions for deletion.
1035 DelInstrs.push_back(Elt: Leaf);
1036 if (IsILPReassociate)
1037 DelInstrs.push_back(Elt: Prev);
1038 DelInstrs.push_back(Elt: &Root);
1039}
1040
1041// Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1042bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1043 Register &SrcReg, Register &DstReg,
1044 unsigned &SubIdx) const {
1045 switch (MI.getOpcode()) {
1046 default: return false;
1047 case PPC::EXTSW:
1048 case PPC::EXTSW_32:
1049 case PPC::EXTSW_32_64:
1050 SrcReg = MI.getOperand(i: 1).getReg();
1051 DstReg = MI.getOperand(i: 0).getReg();
1052 SubIdx = PPC::sub_32;
1053 return true;
1054 }
1055}
1056
1057Register PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1058 int &FrameIndex) const {
1059 if (llvm::is_contained(Range: getLoadOpcodesForSpillArray(), Element: MI.getOpcode())) {
1060 // Check for the operands added by addFrameReference (the immediate is the
1061 // offset which defaults to 0).
1062 if (MI.getOperand(i: 1).isImm() && !MI.getOperand(i: 1).getImm() &&
1063 MI.getOperand(i: 2).isFI()) {
1064 FrameIndex = MI.getOperand(i: 2).getIndex();
1065 return MI.getOperand(i: 0).getReg();
1066 }
1067 }
1068 return 0;
1069}
1070
1071// For opcodes with the ReMaterializable flag set, this function is called to
1072// verify the instruction is really rematable.
1073bool PPCInstrInfo::isReMaterializableImpl(
1074 const MachineInstr &MI) const {
1075 switch (MI.getOpcode()) {
1076 default:
1077 // Let base implementaion decide.
1078 break;
1079 case PPC::LI:
1080 case PPC::LI8:
1081 case PPC::PLI:
1082 case PPC::PLI8:
1083 case PPC::LIS:
1084 case PPC::LIS8:
1085 case PPC::ADDIStocHA:
1086 case PPC::ADDIStocHA8:
1087 case PPC::ADDItocL:
1088 case PPC::ADDItocL8:
1089 case PPC::LOAD_STACK_GUARD:
1090 case PPC::PPCLdFixedAddr:
1091 case PPC::XXLXORz:
1092 case PPC::XXLXORspz:
1093 case PPC::XXLXORdpz:
1094 case PPC::XXLEQVOnes:
1095 case PPC::XXSPLTI32DX:
1096 case PPC::XXSPLTIW:
1097 case PPC::XXSPLTIDP:
1098 case PPC::V_SET0B:
1099 case PPC::V_SET0H:
1100 case PPC::V_SET0:
1101 case PPC::V_SETALLONESB:
1102 case PPC::V_SETALLONESH:
1103 case PPC::V_SETALLONES:
1104 case PPC::CRSET:
1105 case PPC::CRUNSET:
1106 case PPC::XXSETACCZ:
1107 case PPC::DMXXSETACCZ:
1108 return true;
1109 }
1110 return TargetInstrInfo::isReMaterializableImpl(MI);
1111}
1112
1113Register PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1114 int &FrameIndex) const {
1115 if (llvm::is_contained(Range: getStoreOpcodesForSpillArray(), Element: MI.getOpcode())) {
1116 if (MI.getOperand(i: 1).isImm() && !MI.getOperand(i: 1).getImm() &&
1117 MI.getOperand(i: 2).isFI()) {
1118 FrameIndex = MI.getOperand(i: 2).getIndex();
1119 return MI.getOperand(i: 0).getReg();
1120 }
1121 }
1122 return 0;
1123}
1124
1125MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1126 unsigned OpIdx1,
1127 unsigned OpIdx2) const {
1128 MachineFunction &MF = *MI.getParent()->getParent();
1129
1130 // Normal instructions can be commuted the obvious way.
1131 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1132 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1133 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1134 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1135 // changing the relative order of the mask operands might change what happens
1136 // to the high-bits of the mask (and, thus, the result).
1137
1138 // Cannot commute if it has a non-zero rotate count.
1139 if (MI.getOperand(i: 3).getImm() != 0)
1140 return nullptr;
1141
1142 // If we have a zero rotate count, we have:
1143 // M = mask(MB,ME)
1144 // Op0 = (Op1 & ~M) | (Op2 & M)
1145 // Change this to:
1146 // M = mask((ME+1)&31, (MB-1)&31)
1147 // Op0 = (Op2 & ~M) | (Op1 & M)
1148
1149 // Swap op1/op2
1150 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1151 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1152 Register Reg0 = MI.getOperand(i: 0).getReg();
1153 Register Reg1 = MI.getOperand(i: 1).getReg();
1154 Register Reg2 = MI.getOperand(i: 2).getReg();
1155 unsigned SubReg1 = MI.getOperand(i: 1).getSubReg();
1156 unsigned SubReg2 = MI.getOperand(i: 2).getSubReg();
1157 bool Reg1IsKill = MI.getOperand(i: 1).isKill();
1158 bool Reg2IsKill = MI.getOperand(i: 2).isKill();
1159 bool ChangeReg0 = false;
1160 // If machine instrs are no longer in two-address forms, update
1161 // destination register as well.
1162 if (Reg0 == Reg1) {
1163 // Must be two address instruction (i.e. op1 is tied to op0).
1164 assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
1165 "Expecting a two-address instruction!");
1166 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1167 Reg2IsKill = false;
1168 ChangeReg0 = true;
1169 }
1170
1171 // Masks.
1172 unsigned MB = MI.getOperand(i: 4).getImm();
1173 unsigned ME = MI.getOperand(i: 5).getImm();
1174
1175 // We can't commute a trivial mask (there is no way to represent an all-zero
1176 // mask).
1177 if (MB == 0 && ME == 31)
1178 return nullptr;
1179
1180 if (NewMI) {
1181 // Create a new instruction.
1182 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(i: 0).getReg();
1183 bool Reg0IsDead = MI.getOperand(i: 0).isDead();
1184 return BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: MI.getDesc())
1185 .addReg(RegNo: Reg0, Flags: RegState::Define | getDeadRegState(B: Reg0IsDead))
1186 .addReg(RegNo: Reg2, Flags: getKillRegState(B: Reg2IsKill))
1187 .addReg(RegNo: Reg1, Flags: getKillRegState(B: Reg1IsKill))
1188 .addImm(Val: (ME + 1) & 31)
1189 .addImm(Val: (MB - 1) & 31);
1190 }
1191
1192 if (ChangeReg0) {
1193 MI.getOperand(i: 0).setReg(Reg2);
1194 MI.getOperand(i: 0).setSubReg(SubReg2);
1195 }
1196 MI.getOperand(i: 2).setReg(Reg1);
1197 MI.getOperand(i: 1).setReg(Reg2);
1198 MI.getOperand(i: 2).setSubReg(SubReg1);
1199 MI.getOperand(i: 1).setSubReg(SubReg2);
1200 MI.getOperand(i: 2).setIsKill(Reg1IsKill);
1201 MI.getOperand(i: 1).setIsKill(Reg2IsKill);
1202
1203 // Swap the mask around.
1204 MI.getOperand(i: 4).setImm((ME + 1) & 31);
1205 MI.getOperand(i: 5).setImm((MB - 1) & 31);
1206 return &MI;
1207}
1208
1209bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1210 unsigned &SrcOpIdx1,
1211 unsigned &SrcOpIdx2) const {
1212 // For VSX A-Type FMA instructions, it is the first two operands that can be
1213 // commuted, however, because the non-encoded tied input operand is listed
1214 // first, the operands to swap are actually the second and third.
1215
1216 int AltOpc = PPC::getAltVSXFMAOpcode(Opcode: MI.getOpcode());
1217 if (AltOpc == -1)
1218 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1219
1220 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1221 // and SrcOpIdx2.
1222 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
1223}
1224
1225void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
1226 MachineBasicBlock::iterator MI) const {
1227 // This function is used for scheduling, and the nop wanted here is the type
1228 // that terminates dispatch groups on the POWER cores.
1229 unsigned Directive = Subtarget.getCPUDirective();
1230 unsigned Opcode;
1231 switch (Directive) {
1232 default: Opcode = PPC::NOP; break;
1233 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1234 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1235 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1236 // FIXME: Update when POWER9 scheduling model is ready.
1237 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1238 }
1239
1240 DebugLoc DL;
1241 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode));
1242}
1243
1244/// Return the noop instruction to use for a noop.
1245MCInst PPCInstrInfo::getNop() const {
1246 MCInst Nop;
1247 Nop.setOpcode(PPC::NOP);
1248 return Nop;
1249}
1250
1251// Branch analysis.
1252// Note: If the condition register is set to CTR or CTR8 then this is a
1253// BDNZ (imm == 1) or BDZ (imm == 0) branch.
1254bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1255 MachineBasicBlock *&TBB,
1256 MachineBasicBlock *&FBB,
1257 SmallVectorImpl<MachineOperand> &Cond,
1258 bool AllowModify) const {
1259 bool isPPC64 = Subtarget.isPPC64();
1260
1261 // If the block has no terminators, it just falls into the block after it.
1262 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1263 if (I == MBB.end())
1264 return false;
1265
1266 if (!isUnpredicatedTerminator(MI: *I))
1267 return false;
1268
1269 if (AllowModify) {
1270 // If the BB ends with an unconditional branch to the fallthrough BB,
1271 // we eliminate the branch instruction.
1272 if (I->getOpcode() == PPC::B &&
1273 MBB.isLayoutSuccessor(MBB: I->getOperand(i: 0).getMBB())) {
1274 I->eraseFromParent();
1275
1276 // We update iterator after deleting the last branch.
1277 I = MBB.getLastNonDebugInstr();
1278 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1279 return false;
1280 }
1281 }
1282
1283 // Get the last instruction in the block.
1284 MachineInstr &LastInst = *I;
1285
1286 // If there is only one terminator instruction, process it.
1287 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
1288 if (LastInst.getOpcode() == PPC::B) {
1289 if (!LastInst.getOperand(i: 0).isMBB())
1290 return true;
1291 TBB = LastInst.getOperand(i: 0).getMBB();
1292 return false;
1293 } else if (LastInst.getOpcode() == PPC::BCC) {
1294 if (!LastInst.getOperand(i: 2).isMBB())
1295 return true;
1296 // Block ends with fall-through condbranch.
1297 TBB = LastInst.getOperand(i: 2).getMBB();
1298 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1299 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1300 return false;
1301 } else if (LastInst.getOpcode() == PPC::BC) {
1302 if (!LastInst.getOperand(i: 1).isMBB())
1303 return true;
1304 // Block ends with fall-through condbranch.
1305 TBB = LastInst.getOperand(i: 1).getMBB();
1306 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
1307 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1308 return false;
1309 } else if (LastInst.getOpcode() == PPC::BCn) {
1310 if (!LastInst.getOperand(i: 1).isMBB())
1311 return true;
1312 // Block ends with fall-through condbranch.
1313 TBB = LastInst.getOperand(i: 1).getMBB();
1314 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_UNSET));
1315 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1316 return false;
1317 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1318 LastInst.getOpcode() == PPC::BDNZ) {
1319 if (!LastInst.getOperand(i: 0).isMBB())
1320 return true;
1321 if (DisableCTRLoopAnal)
1322 return true;
1323 TBB = LastInst.getOperand(i: 0).getMBB();
1324 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 1));
1325 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1326 isDef: true));
1327 return false;
1328 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1329 LastInst.getOpcode() == PPC::BDZ) {
1330 if (!LastInst.getOperand(i: 0).isMBB())
1331 return true;
1332 if (DisableCTRLoopAnal)
1333 return true;
1334 TBB = LastInst.getOperand(i: 0).getMBB();
1335 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
1336 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1337 isDef: true));
1338 return false;
1339 }
1340
1341 // Otherwise, don't know what this is.
1342 return true;
1343 }
1344
1345 // Get the instruction before it if it's a terminator.
1346 MachineInstr &SecondLastInst = *I;
1347
1348 // If there are three terminators, we don't know what sort of block this is.
1349 if (I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
1350 return true;
1351
1352 // If the block ends with PPC::B and PPC:BCC, handle it.
1353 if (SecondLastInst.getOpcode() == PPC::BCC &&
1354 LastInst.getOpcode() == PPC::B) {
1355 if (!SecondLastInst.getOperand(i: 2).isMBB() ||
1356 !LastInst.getOperand(i: 0).isMBB())
1357 return true;
1358 TBB = SecondLastInst.getOperand(i: 2).getMBB();
1359 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1360 Cond.push_back(Elt: SecondLastInst.getOperand(i: 1));
1361 FBB = LastInst.getOperand(i: 0).getMBB();
1362 return false;
1363 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1364 LastInst.getOpcode() == PPC::B) {
1365 if (!SecondLastInst.getOperand(i: 1).isMBB() ||
1366 !LastInst.getOperand(i: 0).isMBB())
1367 return true;
1368 TBB = SecondLastInst.getOperand(i: 1).getMBB();
1369 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
1370 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1371 FBB = LastInst.getOperand(i: 0).getMBB();
1372 return false;
1373 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1374 LastInst.getOpcode() == PPC::B) {
1375 if (!SecondLastInst.getOperand(i: 1).isMBB() ||
1376 !LastInst.getOperand(i: 0).isMBB())
1377 return true;
1378 TBB = SecondLastInst.getOperand(i: 1).getMBB();
1379 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_UNSET));
1380 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1381 FBB = LastInst.getOperand(i: 0).getMBB();
1382 return false;
1383 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1384 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1385 LastInst.getOpcode() == PPC::B) {
1386 if (!SecondLastInst.getOperand(i: 0).isMBB() ||
1387 !LastInst.getOperand(i: 0).isMBB())
1388 return true;
1389 if (DisableCTRLoopAnal)
1390 return true;
1391 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1392 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 1));
1393 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1394 isDef: true));
1395 FBB = LastInst.getOperand(i: 0).getMBB();
1396 return false;
1397 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1398 SecondLastInst.getOpcode() == PPC::BDZ) &&
1399 LastInst.getOpcode() == PPC::B) {
1400 if (!SecondLastInst.getOperand(i: 0).isMBB() ||
1401 !LastInst.getOperand(i: 0).isMBB())
1402 return true;
1403 if (DisableCTRLoopAnal)
1404 return true;
1405 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1406 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
1407 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1408 isDef: true));
1409 FBB = LastInst.getOperand(i: 0).getMBB();
1410 return false;
1411 }
1412
1413 // If the block ends with two PPC:Bs, handle it. The second one is not
1414 // executed, so remove it.
1415 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1416 if (!SecondLastInst.getOperand(i: 0).isMBB())
1417 return true;
1418 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1419 I = LastInst;
1420 if (AllowModify)
1421 I->eraseFromParent();
1422 return false;
1423 }
1424
1425 // Otherwise, can't handle this.
1426 return true;
1427}
1428
1429unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1430 int *BytesRemoved) const {
1431 assert(!BytesRemoved && "code size not handled");
1432
1433 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1434 if (I == MBB.end())
1435 return 0;
1436
1437 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1438 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1439 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1440 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1441 return 0;
1442
1443 // Remove the branch.
1444 I->eraseFromParent();
1445
1446 I = MBB.end();
1447
1448 if (I == MBB.begin()) return 1;
1449 --I;
1450 if (I->getOpcode() != PPC::BCC &&
1451 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1452 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1453 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1454 return 1;
1455
1456 // Remove the branch.
1457 I->eraseFromParent();
1458 return 2;
1459}
1460
1461unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1462 MachineBasicBlock *TBB,
1463 MachineBasicBlock *FBB,
1464 ArrayRef<MachineOperand> Cond,
1465 const DebugLoc &DL,
1466 int *BytesAdded) const {
1467 // Shouldn't be a fall through.
1468 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1469 assert((Cond.size() == 2 || Cond.size() == 0) &&
1470 "PPC branch conditions have two components!");
1471 assert(!BytesAdded && "code size not handled");
1472
1473 bool isPPC64 = Subtarget.isPPC64();
1474
1475 // One-way branch.
1476 if (!FBB) {
1477 if (Cond.empty()) // Unconditional branch
1478 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::B)).addMBB(MBB: TBB);
1479 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1480 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm() ?
1481 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1482 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(MBB: TBB);
1483 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1484 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BC)).add(MO: Cond[1]).addMBB(MBB: TBB);
1485 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1486 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCn)).add(MO: Cond[1]).addMBB(MBB: TBB);
1487 else // Conditional branch
1488 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCC))
1489 .addImm(Val: Cond[0].getImm())
1490 .add(MO: Cond[1])
1491 .addMBB(MBB: TBB);
1492 return 1;
1493 }
1494
1495 // Two-way Conditional Branch.
1496 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1497 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm() ?
1498 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1499 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(MBB: TBB);
1500 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1501 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BC)).add(MO: Cond[1]).addMBB(MBB: TBB);
1502 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1503 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCn)).add(MO: Cond[1]).addMBB(MBB: TBB);
1504 else
1505 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCC))
1506 .addImm(Val: Cond[0].getImm())
1507 .add(MO: Cond[1])
1508 .addMBB(MBB: TBB);
1509 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::B)).addMBB(MBB: FBB);
1510 return 2;
1511}
1512
1513// Select analysis.
1514bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1515 ArrayRef<MachineOperand> Cond,
1516 Register DstReg, Register TrueReg,
1517 Register FalseReg, int &CondCycles,
1518 int &TrueCycles, int &FalseCycles) const {
1519 if (!Subtarget.hasISEL())
1520 return false;
1521
1522 if (Cond.size() != 2)
1523 return false;
1524
1525 // If this is really a bdnz-like condition, then it cannot be turned into a
1526 // select.
1527 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1528 return false;
1529
1530 // If the conditional branch uses a physical register, then it cannot be
1531 // turned into a select.
1532 if (Cond[1].getReg().isPhysical())
1533 return false;
1534
1535 // Check register classes.
1536 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1537 const TargetRegisterClass *RC =
1538 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1539 if (!RC)
1540 return false;
1541
1542 // isel is for regular integer GPRs only.
1543 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1544 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1545 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1546 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1547 return false;
1548
1549 // FIXME: These numbers are for the A2, how well they work for other cores is
1550 // an open question. On the A2, the isel instruction has a 2-cycle latency
1551 // but single-cycle throughput. These numbers are used in combination with
1552 // the MispredictPenalty setting from the active SchedMachineModel.
1553 CondCycles = 1;
1554 TrueCycles = 1;
1555 FalseCycles = 1;
1556
1557 return true;
1558}
1559
1560void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1561 MachineBasicBlock::iterator MI,
1562 const DebugLoc &dl, Register DestReg,
1563 ArrayRef<MachineOperand> Cond, Register TrueReg,
1564 Register FalseReg) const {
1565 assert(Cond.size() == 2 &&
1566 "PPC branch conditions have two components!");
1567
1568 // Get the register classes.
1569 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1570 const TargetRegisterClass *RC =
1571 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1572 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1573
1574 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1575 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1576 assert((Is64Bit ||
1577 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1578 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1579 "isel is for regular integer GPRs only");
1580
1581 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1582 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1583
1584 unsigned SubIdx = 0;
1585 bool SwapOps = false;
1586 switch (SelectPred) {
1587 case PPC::PRED_EQ:
1588 case PPC::PRED_EQ_MINUS:
1589 case PPC::PRED_EQ_PLUS:
1590 SubIdx = PPC::sub_eq; SwapOps = false; break;
1591 case PPC::PRED_NE:
1592 case PPC::PRED_NE_MINUS:
1593 case PPC::PRED_NE_PLUS:
1594 SubIdx = PPC::sub_eq; SwapOps = true; break;
1595 case PPC::PRED_LT:
1596 case PPC::PRED_LT_MINUS:
1597 case PPC::PRED_LT_PLUS:
1598 SubIdx = PPC::sub_lt; SwapOps = false; break;
1599 case PPC::PRED_GE:
1600 case PPC::PRED_GE_MINUS:
1601 case PPC::PRED_GE_PLUS:
1602 SubIdx = PPC::sub_lt; SwapOps = true; break;
1603 case PPC::PRED_GT:
1604 case PPC::PRED_GT_MINUS:
1605 case PPC::PRED_GT_PLUS:
1606 SubIdx = PPC::sub_gt; SwapOps = false; break;
1607 case PPC::PRED_LE:
1608 case PPC::PRED_LE_MINUS:
1609 case PPC::PRED_LE_PLUS:
1610 SubIdx = PPC::sub_gt; SwapOps = true; break;
1611 case PPC::PRED_UN:
1612 case PPC::PRED_UN_MINUS:
1613 case PPC::PRED_UN_PLUS:
1614 SubIdx = PPC::sub_un; SwapOps = false; break;
1615 case PPC::PRED_NU:
1616 case PPC::PRED_NU_MINUS:
1617 case PPC::PRED_NU_PLUS:
1618 SubIdx = PPC::sub_un; SwapOps = true; break;
1619 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1620 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1621 }
1622
1623 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1624 SecondReg = SwapOps ? TrueReg : FalseReg;
1625
1626 // The first input register of isel cannot be r0. If it is a member
1627 // of a register class that can be r0, then copy it first (the
1628 // register allocator should eliminate the copy).
1629 if (MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::R0) ||
1630 MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::X0)) {
1631 const TargetRegisterClass *FirstRC =
1632 MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::X0) ?
1633 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1634 Register OldFirstReg = FirstReg;
1635 FirstReg = MRI.createVirtualRegister(RegClass: FirstRC);
1636 BuildMI(BB&: MBB, I: MI, MIMD: dl, MCID: get(Opcode: TargetOpcode::COPY), DestReg: FirstReg)
1637 .addReg(RegNo: OldFirstReg);
1638 }
1639
1640 BuildMI(BB&: MBB, I: MI, MIMD: dl, MCID: get(Opcode: OpCode), DestReg)
1641 .addReg(RegNo: FirstReg)
1642 .addReg(RegNo: SecondReg)
1643 .addReg(RegNo: Cond[1].getReg(), Flags: {}, SubReg: SubIdx);
1644}
1645
1646static unsigned getCRBitValue(unsigned CRBit) {
1647 unsigned Ret = 4;
1648 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1649 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1650 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1651 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1652 Ret = 3;
1653 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1654 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1655 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1656 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1657 Ret = 2;
1658 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1659 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1660 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1661 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1662 Ret = 1;
1663 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1664 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1665 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1666 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1667 Ret = 0;
1668
1669 assert(Ret != 4 && "Invalid CR bit register");
1670 return Ret;
1671}
1672
1673void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1674 MachineBasicBlock::iterator I,
1675 const DebugLoc &DL, Register DestReg,
1676 Register SrcReg, bool KillSrc,
1677 bool RenamableDest, bool RenamableSrc) const {
1678 // We can end up with self copies and similar things as a result of VSX copy
1679 // legalization. Promote them here.
1680 if (PPC::F8RCRegClass.contains(Reg: DestReg) &&
1681 PPC::VSRCRegClass.contains(Reg: SrcReg)) {
1682 MCRegister SuperReg =
1683 RI.getMatchingSuperReg(Reg: DestReg, SubIdx: PPC::sub_64, RC: &PPC::VSRCRegClass);
1684
1685 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1686 llvm_unreachable("nop VSX copy");
1687
1688 DestReg = SuperReg;
1689 } else if (PPC::F8RCRegClass.contains(Reg: SrcReg) &&
1690 PPC::VSRCRegClass.contains(Reg: DestReg)) {
1691 MCRegister SuperReg =
1692 RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: PPC::sub_64, RC: &PPC::VSRCRegClass);
1693
1694 if (VSXSelfCopyCrash && DestReg == SuperReg)
1695 llvm_unreachable("nop VSX copy");
1696
1697 SrcReg = SuperReg;
1698 }
1699
1700 // Different class register copy
1701 if (PPC::CRBITRCRegClass.contains(Reg: SrcReg) &&
1702 PPC::GPRCRegClass.contains(Reg: DestReg)) {
1703 MCRegister CRReg = getCRFromCRBit(SrcReg);
1704 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MFOCRF), DestReg).addReg(RegNo: CRReg);
1705 getKillRegState(B: KillSrc);
1706 // Rotate the CR bit in the CR fields to be the least significant bit and
1707 // then mask with 0x1 (MB = ME = 31).
1708 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::RLWINM), DestReg)
1709 .addReg(RegNo: DestReg, Flags: RegState::Kill)
1710 .addImm(Val: RI.getEncodingValue(Reg: CRReg) * 4 + (4 - getCRBitValue(CRBit: SrcReg)))
1711 .addImm(Val: 31)
1712 .addImm(Val: 31);
1713 return;
1714 } else if (PPC::CRRCRegClass.contains(Reg: SrcReg) &&
1715 (PPC::G8RCRegClass.contains(Reg: DestReg) ||
1716 PPC::GPRCRegClass.contains(Reg: DestReg))) {
1717 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: DestReg);
1718 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1719 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1720 unsigned CRNum = RI.getEncodingValue(Reg: SrcReg);
1721 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: MvCode), DestReg).addReg(RegNo: SrcReg);
1722 getKillRegState(B: KillSrc);
1723 if (CRNum == 7)
1724 return;
1725 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1726 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ShCode), DestReg)
1727 .addReg(RegNo: DestReg, Flags: RegState::Kill)
1728 .addImm(Val: CRNum * 4 + 4)
1729 .addImm(Val: 28)
1730 .addImm(Val: 31);
1731 return;
1732 } else if (PPC::G8RCRegClass.contains(Reg: SrcReg) &&
1733 PPC::VSFRCRegClass.contains(Reg: DestReg)) {
1734 assert(Subtarget.hasDirectMove() &&
1735 "Subtarget doesn't support directmove, don't know how to copy.");
1736 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MTVSRD), DestReg).addReg(RegNo: SrcReg);
1737 NumGPRtoVSRSpill++;
1738 getKillRegState(B: KillSrc);
1739 return;
1740 } else if (PPC::VSFRCRegClass.contains(Reg: SrcReg) &&
1741 PPC::G8RCRegClass.contains(Reg: DestReg)) {
1742 assert(Subtarget.hasDirectMove() &&
1743 "Subtarget doesn't support directmove, don't know how to copy.");
1744 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MFVSRD), DestReg).addReg(RegNo: SrcReg);
1745 getKillRegState(B: KillSrc);
1746 return;
1747 } else if (PPC::SPERCRegClass.contains(Reg: SrcReg) &&
1748 PPC::GPRCRegClass.contains(Reg: DestReg)) {
1749 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::EFSCFD), DestReg).addReg(RegNo: SrcReg);
1750 getKillRegState(B: KillSrc);
1751 return;
1752 } else if (PPC::GPRCRegClass.contains(Reg: SrcReg) &&
1753 PPC::SPERCRegClass.contains(Reg: DestReg)) {
1754 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::EFDCFS), DestReg).addReg(RegNo: SrcReg);
1755 getKillRegState(B: KillSrc);
1756 return;
1757 } else if ((PPC::G8RCRegClass.contains(Reg: DestReg) ||
1758 PPC::GPRCRegClass.contains(Reg: DestReg)) &&
1759 SrcReg == PPC::CARRY) {
1760 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: DestReg);
1761 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Is64Bit ? PPC::MFSPR8 : PPC::MFSPR), DestReg)
1762 .addImm(Val: 1)
1763 .addReg(RegNo: PPC::CARRY, Flags: RegState::Implicit);
1764 return;
1765 } else if ((PPC::G8RCRegClass.contains(Reg: SrcReg) ||
1766 PPC::GPRCRegClass.contains(Reg: SrcReg)) &&
1767 DestReg == PPC::CARRY) {
1768 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: SrcReg);
1769 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Is64Bit ? PPC::MTSPR8 : PPC::MTSPR))
1770 .addImm(Val: 1)
1771 .addReg(RegNo: SrcReg)
1772 .addReg(RegNo: PPC::CARRY, Flags: RegState::ImplicitDefine);
1773 return;
1774 }
1775
1776 unsigned Opc;
1777 if (PPC::GPRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1778 Opc = PPC::OR;
1779 else if (PPC::G8RCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1780 Opc = PPC::OR8;
1781 else if (PPC::F4RCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1782 Opc = PPC::FMR;
1783 else if (PPC::CRRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1784 Opc = PPC::MCRF;
1785 else if (PPC::VRRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1786 Opc = PPC::VOR;
1787 else if (PPC::VSRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1788 // There are two different ways this can be done:
1789 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1790 // issue in VSU pipeline 0.
1791 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1792 // can go to either pipeline.
1793 // We'll always use xxlor here, because in practically all cases where
1794 // copies are generated, they are close enough to some use that the
1795 // lower-latency form is preferable.
1796 Opc = PPC::XXLOR;
1797 else if (PPC::VSFRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg) ||
1798 PPC::VSSRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1799 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1800 else if (Subtarget.pairedVectorMemops() &&
1801 PPC::VSRpRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
1802 if (SrcReg > PPC::VSRp15)
1803 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1804 else
1805 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1806 if (DestReg > PPC::VSRp15)
1807 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1808 else
1809 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1810 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg).
1811 addReg(RegNo: SrcReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1812 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: DestReg + 1).
1813 addReg(RegNo: SrcReg + 1).addReg(RegNo: SrcReg + 1, Flags: getKillRegState(B: KillSrc));
1814 return;
1815 }
1816 else if (PPC::CRBITRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1817 Opc = PPC::CROR;
1818 else if (PPC::SPERCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1819 Opc = PPC::EVOR;
1820 else if ((PPC::ACCRCRegClass.contains(Reg: DestReg) ||
1821 PPC::UACCRCRegClass.contains(Reg: DestReg)) &&
1822 (PPC::ACCRCRegClass.contains(Reg: SrcReg) ||
1823 PPC::UACCRCRegClass.contains(Reg: SrcReg))) {
1824 // If primed, de-prime the source register, copy the individual registers
1825 // and prime the destination if needed. The vector subregisters are
1826 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1827 // source is primed, we need to re-prime it after the copy as well.
1828 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1829 bool DestPrimed = PPC::ACCRCRegClass.contains(Reg: DestReg);
1830 bool SrcPrimed = PPC::ACCRCRegClass.contains(Reg: SrcReg);
1831 MCRegister VSLSrcReg =
1832 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1833 MCRegister VSLDestReg =
1834 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1835 if (SrcPrimed)
1836 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMFACC), DestReg: SrcReg).addReg(RegNo: SrcReg);
1837 for (unsigned Idx = 0; Idx < 4; Idx++)
1838 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: VSLDestReg + Idx)
1839 .addReg(RegNo: VSLSrcReg + Idx)
1840 .addReg(RegNo: VSLSrcReg + Idx, Flags: getKillRegState(B: KillSrc));
1841 if (DestPrimed)
1842 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMTACC), DestReg).addReg(RegNo: DestReg);
1843 if (SrcPrimed && !KillSrc)
1844 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMTACC), DestReg: SrcReg).addReg(RegNo: SrcReg);
1845 return;
1846 } else if (PPC::G8pRCRegClass.contains(Reg: DestReg) &&
1847 PPC::G8pRCRegClass.contains(Reg: SrcReg)) {
1848 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1849 unsigned DestRegIdx = DestReg - PPC::G8p0;
1850 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1851 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1852 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1853 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1854 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1855 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DestRegSub0)
1856 .addReg(RegNo: SrcRegSub0)
1857 .addReg(RegNo: SrcRegSub0, Flags: getKillRegState(B: KillSrc));
1858 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DestRegSub1)
1859 .addReg(RegNo: SrcRegSub1)
1860 .addReg(RegNo: SrcRegSub1, Flags: getKillRegState(B: KillSrc));
1861 return;
1862 } else if ((PPC::WACCRCRegClass.contains(Reg: DestReg) ||
1863 PPC::WACC_HIRCRegClass.contains(Reg: DestReg)) &&
1864 (PPC::WACCRCRegClass.contains(Reg: SrcReg) ||
1865 PPC::WACC_HIRCRegClass.contains(Reg: SrcReg))) {
1866
1867 Opc = PPC::WACCRCRegClass.contains(Reg: SrcReg) ? PPC::DMXXEXTFDMR512
1868 : PPC::DMXXEXTFDMR512_HI;
1869
1870 RegScavenger RS;
1871 RS.enterBasicBlockEnd(MBB);
1872 RS.backward(I: std::next(x: I));
1873
1874 Register TmpReg1 = RS.scavengeRegisterBackwards(RC: PPC::VSRpRCRegClass, To: I,
1875 /* RestoreAfter */ false, SPAdj: 0,
1876 /* AllowSpill */ false);
1877
1878 RS.setRegUsed(Reg: TmpReg1);
1879 Register TmpReg2 = RS.scavengeRegisterBackwards(RC: PPC::VSRpRCRegClass, To: I,
1880 /* RestoreAfter */ false, SPAdj: 0,
1881 /* AllowSpill */ false);
1882
1883 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc))
1884 .addReg(RegNo: TmpReg1, Flags: RegState::Define)
1885 .addReg(RegNo: TmpReg2, Flags: RegState::Define)
1886 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1887
1888 Opc = PPC::WACCRCRegClass.contains(Reg: DestReg) ? PPC::DMXXINSTDMR512
1889 : PPC::DMXXINSTDMR512_HI;
1890
1891 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
1892 .addReg(RegNo: TmpReg1, Flags: RegState::Kill)
1893 .addReg(RegNo: TmpReg2, Flags: RegState::Kill);
1894
1895 return;
1896 } else if (PPC::DMRRCRegClass.contains(Reg: DestReg) &&
1897 PPC::DMRRCRegClass.contains(Reg: SrcReg)) {
1898
1899 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::DMMR), DestReg)
1900 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1901
1902 return;
1903
1904 } else
1905 llvm_unreachable("Impossible reg-to-reg copy");
1906
1907 const MCInstrDesc &MCID = get(Opcode: Opc);
1908 if (MCID.getNumOperands() == 3)
1909 BuildMI(BB&: MBB, I, MIMD: DL, MCID, DestReg)
1910 .addReg(RegNo: SrcReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1911 else
1912 BuildMI(BB&: MBB, I, MIMD: DL, MCID, DestReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1913}
1914
1915unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1916 int OpcodeIndex = 0;
1917
1918 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1919 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1920 OpcodeIndex = SOK_Int4Spill;
1921 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1922 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1923 OpcodeIndex = SOK_Int8Spill;
1924 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1925 OpcodeIndex = SOK_Float8Spill;
1926 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1927 OpcodeIndex = SOK_Float4Spill;
1928 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1929 OpcodeIndex = SOK_SPESpill;
1930 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1931 OpcodeIndex = SOK_CRSpill;
1932 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1933 OpcodeIndex = SOK_CRBitSpill;
1934 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1935 OpcodeIndex = SOK_VRVectorSpill;
1936 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1937 OpcodeIndex = SOK_VSXVectorSpill;
1938 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1939 OpcodeIndex = SOK_VectorFloat8Spill;
1940 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1941 OpcodeIndex = SOK_VectorFloat4Spill;
1942 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1943 OpcodeIndex = SOK_SpillToVSR;
1944 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1945 assert(Subtarget.pairedVectorMemops() &&
1946 "Register unexpected when paired memops are disabled.");
1947 OpcodeIndex = SOK_AccumulatorSpill;
1948 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1949 assert(Subtarget.pairedVectorMemops() &&
1950 "Register unexpected when paired memops are disabled.");
1951 OpcodeIndex = SOK_UAccumulatorSpill;
1952 } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1953 assert(Subtarget.pairedVectorMemops() &&
1954 "Register unexpected when paired memops are disabled.");
1955 OpcodeIndex = SOK_WAccumulatorSpill;
1956 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1957 assert(Subtarget.pairedVectorMemops() &&
1958 "Register unexpected when paired memops are disabled.");
1959 OpcodeIndex = SOK_PairedVecSpill;
1960 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1961 OpcodeIndex = SOK_PairedG8Spill;
1962 } else if (PPC::DMRROWRCRegClass.hasSubClassEq(RC)) {
1963 llvm_unreachable("TODO: Implement spill DMRROW regclass!");
1964 } else if (PPC::DMRROWpRCRegClass.hasSubClassEq(RC)) {
1965 llvm_unreachable("TODO: Implement spill DMRROWp regclass!");
1966 } else if (PPC::DMRpRCRegClass.hasSubClassEq(RC)) {
1967 OpcodeIndex = SOK_DMRpSpill;
1968 } else if (PPC::DMRRCRegClass.hasSubClassEq(RC)) {
1969 OpcodeIndex = SOK_DMRSpill;
1970 } else {
1971 llvm_unreachable("Unknown regclass!");
1972 }
1973 return OpcodeIndex;
1974}
1975
1976unsigned
1977PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1978 ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1979 return OpcodesForSpill[getSpillIndex(RC)];
1980}
1981
1982unsigned
1983PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1984 ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1985 return OpcodesForSpill[getSpillIndex(RC)];
1986}
1987
1988void PPCInstrInfo::StoreRegToStackSlot(
1989 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1990 const TargetRegisterClass *RC,
1991 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1992 unsigned Opcode = getStoreOpcodeForSpill(RC);
1993 DebugLoc DL;
1994
1995 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1996 FuncInfo->setHasSpills();
1997
1998 NewMIs.push_back(Elt: addFrameReference(
1999 MIB: BuildMI(MF, MIMD: DL, MCID: get(Opcode)).addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)),
2000 FI: FrameIdx));
2001
2002 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
2003 PPC::CRBITRCRegClass.hasSubClassEq(RC))
2004 FuncInfo->setSpillsCR();
2005
2006 if (isXFormMemOp(Opcode))
2007 FuncInfo->setHasNonRISpills();
2008}
2009
2010void PPCInstrInfo::storeRegToStackSlotNoUpd(
2011 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
2012 bool isKill, int FrameIdx, const TargetRegisterClass *RC) const {
2013 MachineFunction &MF = *MBB.getParent();
2014 SmallVector<MachineInstr *, 4> NewMIs;
2015
2016 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
2017
2018 for (MachineInstr *NewMI : NewMIs)
2019 MBB.insert(I: MI, MI: NewMI);
2020
2021 const MachineFrameInfo &MFI = MF.getFrameInfo();
2022 MachineMemOperand *MMO = MF.getMachineMemOperand(
2023 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
2024 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
2025 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
2026 NewMIs.back()->addMemOperand(MF, MO: MMO);
2027}
2028
2029void PPCInstrInfo::storeRegToStackSlot(
2030 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
2031 bool isKill, int FrameIdx, const TargetRegisterClass *RC, Register VReg,
2032 MachineInstr::MIFlag Flags) const {
2033 // We need to avoid a situation in which the value from a VRRC register is
2034 // spilled using an Altivec instruction and reloaded into a VSRC register
2035 // using a VSX instruction. The issue with this is that the VSX
2036 // load/store instructions swap the doublewords in the vector and the Altivec
2037 // ones don't. The register classes on the spill/reload may be different if
2038 // the register is defined using an Altivec instruction and is then used by a
2039 // VSX instruction.
2040 RC = updatedRC(RC);
2041 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC);
2042}
2043
2044void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
2045 unsigned DestReg, int FrameIdx,
2046 const TargetRegisterClass *RC,
2047 SmallVectorImpl<MachineInstr *> &NewMIs)
2048 const {
2049 unsigned Opcode = getLoadOpcodeForSpill(RC);
2050 NewMIs.push_back(Elt: addFrameReference(MIB: BuildMI(MF, MIMD: DL, MCID: get(Opcode), DestReg),
2051 FI: FrameIdx));
2052}
2053
2054void PPCInstrInfo::loadRegFromStackSlotNoUpd(
2055 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
2056 int FrameIdx, const TargetRegisterClass *RC) const {
2057 MachineFunction &MF = *MBB.getParent();
2058 SmallVector<MachineInstr*, 4> NewMIs;
2059 DebugLoc DL;
2060 if (MI != MBB.end()) DL = MI->getDebugLoc();
2061
2062 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2063
2064 for (MachineInstr *NewMI : NewMIs)
2065 MBB.insert(I: MI, MI: NewMI);
2066
2067 const MachineFrameInfo &MFI = MF.getFrameInfo();
2068 MachineMemOperand *MMO = MF.getMachineMemOperand(
2069 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
2070 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
2071 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
2072 NewMIs.back()->addMemOperand(MF, MO: MMO);
2073}
2074
2075void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
2076 MachineBasicBlock::iterator MI,
2077 Register DestReg, int FrameIdx,
2078 const TargetRegisterClass *RC,
2079 Register VReg, unsigned SubReg,
2080 MachineInstr::MIFlag Flags) const {
2081 // We need to avoid a situation in which the value from a VRRC register is
2082 // spilled using an Altivec instruction and reloaded into a VSRC register
2083 // using a VSX instruction. The issue with this is that the VSX
2084 // load/store instructions swap the doublewords in the vector and the Altivec
2085 // ones don't. The register classes on the spill/reload may be different if
2086 // the register is defined using an Altivec instruction and is then used by a
2087 // VSX instruction.
2088 RC = updatedRC(RC);
2089
2090 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC);
2091}
2092
2093bool PPCInstrInfo::
2094reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
2095 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2096 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2097 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2098 else
2099 // Leave the CR# the same, but invert the condition.
2100 Cond[0].setImm(PPC::InvertPredicate(Opcode: (PPC::Predicate)Cond[0].getImm()));
2101 return false;
2102}
2103
2104// For some instructions, it is legal to fold ZERO into the RA register field.
2105// This function performs that fold by replacing the operand with PPC::ZERO,
2106// it does not consider whether the load immediate zero is no longer in use.
2107bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2108 Register Reg) const {
2109 // A zero immediate should always be loaded with a single li.
2110 unsigned DefOpc = DefMI.getOpcode();
2111 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2112 return false;
2113 if (!DefMI.getOperand(i: 1).isImm())
2114 return false;
2115 if (DefMI.getOperand(i: 1).getImm() != 0)
2116 return false;
2117
2118 // Note that we cannot here invert the arguments of an isel in order to fold
2119 // a ZERO into what is presented as the second argument. All we have here
2120 // is the condition bit, and that might come from a CR-logical bit operation.
2121
2122 const MCInstrDesc &UseMCID = UseMI.getDesc();
2123
2124 // Only fold into real machine instructions.
2125 if (UseMCID.isPseudo())
2126 return false;
2127
2128 // We need to find which of the User's operands is to be folded, that will be
2129 // the operand that matches the given register ID.
2130 unsigned UseIdx;
2131 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2132 if (UseMI.getOperand(i: UseIdx).isReg() &&
2133 UseMI.getOperand(i: UseIdx).getReg() == Reg)
2134 break;
2135
2136 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2137 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2138
2139 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2140 // register (which might also be specified as a pointer class kind).
2141
2142 const MCOperandInfo &UseInfo = UseMCID.operands()[UseIdx];
2143 int16_t RegClass = getOpRegClassID(OpInfo: UseInfo);
2144 if (UseInfo.RegClass != PPC::GPRC_NOR0RegClassID &&
2145 UseInfo.RegClass != PPC::G8RC_NOX0RegClassID)
2146 return false;
2147
2148 // Make sure this is not tied to an output register (or otherwise
2149 // constrained). This is true for ST?UX registers, for example, which
2150 // are tied to their output registers.
2151 if (UseInfo.Constraints != 0)
2152 return false;
2153
2154 MCRegister ZeroReg =
2155 RegClass == PPC::G8RC_NOX0RegClassID ? PPC::ZERO8 : PPC::ZERO;
2156
2157 LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2158 LLVM_DEBUG(UseMI.dump());
2159 UseMI.getOperand(i: UseIdx).setReg(ZeroReg);
2160 LLVM_DEBUG(dbgs() << "Into: ");
2161 LLVM_DEBUG(UseMI.dump());
2162 return true;
2163}
2164
2165// Folds zero into instructions which have a load immediate zero as an operand
2166// but also recognize zero as immediate zero. If the definition of the load
2167// has no more users it is deleted.
2168bool PPCInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2169 Register Reg, MachineRegisterInfo *MRI) const {
2170 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2171 if (MRI->use_nodbg_empty(RegNo: Reg))
2172 DefMI.eraseFromParent();
2173 return Changed;
2174}
2175
2176static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
2177 for (MachineInstr &MI : MBB)
2178 if (MI.definesRegister(Reg: PPC::CTR, /*TRI=*/nullptr) ||
2179 MI.definesRegister(Reg: PPC::CTR8, /*TRI=*/nullptr))
2180 return true;
2181 return false;
2182}
2183
2184// We should make sure that, if we're going to predicate both sides of a
2185// condition (a diamond), that both sides don't define the counter register. We
2186// can predicate counter-decrement-based branches, but while that predicates
2187// the branching, it does not predicate the counter decrement. If we tried to
2188// merge the triangle into one predicated block, we'd decrement the counter
2189// twice.
2190bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
2191 unsigned NumT, unsigned ExtraT,
2192 MachineBasicBlock &FMBB,
2193 unsigned NumF, unsigned ExtraF,
2194 BranchProbability Probability) const {
2195 return !(MBBDefinesCTR(MBB&: TMBB) && MBBDefinesCTR(MBB&: FMBB));
2196}
2197
2198
2199bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
2200 // The predicated branches are identified by their type, not really by the
2201 // explicit presence of a predicate. Furthermore, some of them can be
2202 // predicated more than once. Because if conversion won't try to predicate
2203 // any instruction which already claims to be predicated (by returning true
2204 // here), always return false. In doing so, we let isPredicable() be the
2205 // final word on whether not the instruction can be (further) predicated.
2206
2207 return false;
2208}
2209
2210bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2211 const MachineBasicBlock *MBB,
2212 const MachineFunction &MF) const {
2213 switch (MI.getOpcode()) {
2214 default:
2215 break;
2216 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2217 // across them, since some FP operations may change content of FPSCR.
2218 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2219 case PPC::MFFS:
2220 case PPC::MTFSF:
2221 case PPC::FENCE:
2222 return true;
2223 }
2224 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
2225}
2226
2227bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
2228 ArrayRef<MachineOperand> Pred) const {
2229 unsigned OpC = MI.getOpcode();
2230 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2231 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2232 bool isPPC64 = Subtarget.isPPC64();
2233 MI.setDesc(get(Opcode: Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2234 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2235 // Need add Def and Use for CTR implicit operand.
2236 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2237 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::Implicit)
2238 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::ImplicitDefine);
2239 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2240 MI.setDesc(get(Opcode: PPC::BCLR));
2241 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2242 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2243 MI.setDesc(get(Opcode: PPC::BCLRn));
2244 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2245 } else {
2246 MI.setDesc(get(Opcode: PPC::BCCLR));
2247 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2248 .addImm(Val: Pred[0].getImm())
2249 .add(MO: Pred[1]);
2250 }
2251
2252 return true;
2253 } else if (OpC == PPC::B) {
2254 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2255 bool isPPC64 = Subtarget.isPPC64();
2256 MI.setDesc(get(Opcode: Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2257 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2258 // Need add Def and Use for CTR implicit operand.
2259 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2260 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::Implicit)
2261 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::ImplicitDefine);
2262 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2263 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2264 MI.removeOperand(OpNo: 0);
2265
2266 MI.setDesc(get(Opcode: PPC::BC));
2267 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2268 .add(MO: Pred[1])
2269 .addMBB(MBB);
2270 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2271 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2272 MI.removeOperand(OpNo: 0);
2273
2274 MI.setDesc(get(Opcode: PPC::BCn));
2275 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2276 .add(MO: Pred[1])
2277 .addMBB(MBB);
2278 } else {
2279 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2280 MI.removeOperand(OpNo: 0);
2281
2282 MI.setDesc(get(Opcode: PPC::BCC));
2283 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2284 .addImm(Val: Pred[0].getImm())
2285 .add(MO: Pred[1])
2286 .addMBB(MBB);
2287 }
2288
2289 return true;
2290 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2291 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2292 OpC == PPC::BCTRL8_RM) {
2293 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2294 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2295
2296 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2297 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2298 bool isPPC64 = Subtarget.isPPC64();
2299
2300 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2301 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2302 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2303 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2304 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2305 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2306 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2307 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2308 } else {
2309 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2310 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2311 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2312 .addImm(Val: Pred[0].getImm())
2313 .add(MO: Pred[1]);
2314 }
2315
2316 // Need add Def and Use for LR implicit operand.
2317 if (setLR)
2318 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2319 .addReg(RegNo: isPPC64 ? PPC::LR8 : PPC::LR, Flags: RegState::Implicit)
2320 .addReg(RegNo: isPPC64 ? PPC::LR8 : PPC::LR, Flags: RegState::ImplicitDefine);
2321 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2322 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2323 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
2324
2325 return true;
2326 }
2327
2328 return false;
2329}
2330
2331bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
2332 ArrayRef<MachineOperand> Pred2) const {
2333 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2334 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2335
2336 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2337 return false;
2338 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2339 return false;
2340
2341 // P1 can only subsume P2 if they test the same condition register.
2342 if (Pred1[1].getReg() != Pred2[1].getReg())
2343 return false;
2344
2345 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2346 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2347
2348 if (P1 == P2)
2349 return true;
2350
2351 // Does P1 subsume P2, e.g. GE subsumes GT.
2352 if (P1 == PPC::PRED_LE &&
2353 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2354 return true;
2355 if (P1 == PPC::PRED_GE &&
2356 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2357 return true;
2358
2359 return false;
2360}
2361
2362bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
2363 std::vector<MachineOperand> &Pred,
2364 bool SkipDead) const {
2365 // Note: At the present time, the contents of Pred from this function is
2366 // unused by IfConversion. This implementation follows ARM by pushing the
2367 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2368 // predicate, instructions defining CTR or CTR8 are also included as
2369 // predicate-defining instructions.
2370
2371 const TargetRegisterClass *RCs[] =
2372 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2373 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2374
2375 bool Found = false;
2376 for (const MachineOperand &MO : MI.operands()) {
2377 for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2378 const TargetRegisterClass *RC = RCs[c];
2379 if (MO.isReg()) {
2380 if (MO.isDef() && RC->contains(Reg: MO.getReg())) {
2381 Pred.push_back(x: MO);
2382 Found = true;
2383 }
2384 } else if (MO.isRegMask()) {
2385 for (MCPhysReg R : *RC)
2386 if (MO.clobbersPhysReg(PhysReg: R)) {
2387 Pred.push_back(x: MO);
2388 Found = true;
2389 }
2390 }
2391 }
2392 }
2393
2394 return Found;
2395}
2396
2397bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2398 Register &SrcReg2, int64_t &Mask,
2399 int64_t &Value) const {
2400 unsigned Opc = MI.getOpcode();
2401
2402 switch (Opc) {
2403 default: return false;
2404 case PPC::CMPWI:
2405 case PPC::CMPLWI:
2406 case PPC::CMPDI:
2407 case PPC::CMPLDI:
2408 SrcReg = MI.getOperand(i: 1).getReg();
2409 SrcReg2 = 0;
2410 Value = MI.getOperand(i: 2).getImm();
2411 Mask = 0xFFFF;
2412 return true;
2413 case PPC::CMPW:
2414 case PPC::CMPLW:
2415 case PPC::CMPD:
2416 case PPC::CMPLD:
2417 case PPC::FCMPUS:
2418 case PPC::FCMPUD:
2419 SrcReg = MI.getOperand(i: 1).getReg();
2420 SrcReg2 = MI.getOperand(i: 2).getReg();
2421 Value = 0;
2422 Mask = 0;
2423 return true;
2424 }
2425}
2426
2427bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
2428 Register SrcReg2, int64_t Mask,
2429 int64_t Value,
2430 const MachineRegisterInfo *MRI) const {
2431 if (DisableCmpOpt)
2432 return false;
2433
2434 int OpC = CmpInstr.getOpcode();
2435 Register CRReg = CmpInstr.getOperand(i: 0).getReg();
2436
2437 // FP record forms set CR1 based on the exception status bits, not a
2438 // comparison with zero.
2439 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2440 return false;
2441
2442 // The record forms set the condition register based on a signed comparison
2443 // with zero (so says the ISA manual). This is not as straightforward as it
2444 // seems, however, because this is always a 64-bit comparison on PPC64, even
2445 // for instructions that are 32-bit in nature (like slw for example).
2446 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2447 // for equality checks (as those don't depend on the sign). On PPC64,
2448 // we are restricted to equality for unsigned 64-bit comparisons and for
2449 // signed 32-bit comparisons the applicability is more restricted.
2450 bool isPPC64 = Subtarget.isPPC64();
2451 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2452 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2453 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2454
2455 // Look through copies unless that gets us to a physical register.
2456 Register ActualSrc = RI.lookThruCopyLike(SrcReg, MRI);
2457 if (ActualSrc.isVirtual())
2458 SrcReg = ActualSrc;
2459
2460 // Get the unique definition of SrcReg.
2461 MachineInstr *MI = MRI->getUniqueVRegDef(Reg: SrcReg);
2462 if (!MI) return false;
2463
2464 bool equalityOnly = false;
2465 bool noSub = false;
2466 if (isPPC64) {
2467 if (is32BitSignedCompare) {
2468 // We can perform this optimization only if SrcReg is sign-extending.
2469 if (isSignExtended(Reg: SrcReg, MRI))
2470 noSub = true;
2471 else
2472 return false;
2473 } else if (is32BitUnsignedCompare) {
2474 // We can perform this optimization, equality only, if SrcReg is
2475 // zero-extending.
2476 if (isZeroExtended(Reg: SrcReg, MRI)) {
2477 noSub = true;
2478 equalityOnly = true;
2479 } else
2480 return false;
2481 } else
2482 equalityOnly = is64BitUnsignedCompare;
2483 } else
2484 equalityOnly = is32BitUnsignedCompare;
2485
2486 if (equalityOnly) {
2487 // We need to check the uses of the condition register in order to reject
2488 // non-equality comparisons.
2489 for (MachineRegisterInfo::use_instr_iterator
2490 I = MRI->use_instr_begin(RegNo: CRReg), IE = MRI->use_instr_end();
2491 I != IE; ++I) {
2492 MachineInstr *UseMI = &*I;
2493 if (UseMI->getOpcode() == PPC::BCC) {
2494 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(i: 0).getImm();
2495 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2496 // We ignore hint bits when checking for non-equality comparisons.
2497 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2498 return false;
2499 } else if (UseMI->getOpcode() == PPC::ISEL ||
2500 UseMI->getOpcode() == PPC::ISEL8) {
2501 unsigned SubIdx = UseMI->getOperand(i: 3).getSubReg();
2502 if (SubIdx != PPC::sub_eq)
2503 return false;
2504 } else
2505 return false;
2506 }
2507 }
2508
2509 MachineBasicBlock::iterator I = CmpInstr;
2510
2511 // Scan forward to find the first use of the compare.
2512 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2513 ++I) {
2514 bool FoundUse = false;
2515 for (MachineRegisterInfo::use_instr_iterator
2516 J = MRI->use_instr_begin(RegNo: CRReg), JE = MRI->use_instr_end();
2517 J != JE; ++J)
2518 if (&*J == &*I) {
2519 FoundUse = true;
2520 break;
2521 }
2522
2523 if (FoundUse)
2524 break;
2525 }
2526
2527 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
2528 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
2529
2530 // There are two possible candidates which can be changed to set CR[01].
2531 // One is MI, the other is a SUB instruction.
2532 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2533 MachineInstr *Sub = nullptr;
2534 if (SrcReg2 != 0)
2535 // MI is not a candidate for CMPrr.
2536 MI = nullptr;
2537 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2538 // same BB as the comparison. This is to allow the check below to avoid calls
2539 // (and other explicit clobbers); instead we should really check for these
2540 // more explicitly (in at least a few predecessors).
2541 else if (MI->getParent() != CmpInstr.getParent())
2542 return false;
2543 else if (Value != 0) {
2544 // The record-form instructions set CR bit based on signed comparison
2545 // against 0. We try to convert a compare against 1 or -1 into a compare
2546 // against 0 to exploit record-form instructions. For example, we change
2547 // the condition "greater than -1" into "greater than or equal to 0"
2548 // and "less than 1" into "less than or equal to 0".
2549
2550 // Since we optimize comparison based on a specific branch condition,
2551 // we don't optimize if condition code is used by more than once.
2552 if (equalityOnly || !MRI->hasOneUse(RegNo: CRReg))
2553 return false;
2554
2555 MachineInstr *UseMI = &*MRI->use_instr_begin(RegNo: CRReg);
2556 if (UseMI->getOpcode() != PPC::BCC)
2557 return false;
2558
2559 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(i: 0).getImm();
2560 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2561 unsigned PredHint = PPC::getPredicateHint(Opcode: Pred);
2562 int16_t Immed = (int16_t)Value;
2563
2564 // When modifying the condition in the predicate, we propagate hint bits
2565 // from the original predicate to the new one.
2566 if (Immed == -1 && PredCond == PPC::PRED_GT)
2567 // We convert "greater than -1" into "greater than or equal to 0",
2568 // since we are assuming signed comparison by !equalityOnly
2569 Pred = PPC::getPredicate(Condition: PPC::PRED_GE, Hint: PredHint);
2570 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2571 // We convert "less than or equal to -1" into "less than 0".
2572 Pred = PPC::getPredicate(Condition: PPC::PRED_LT, Hint: PredHint);
2573 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2574 // We convert "less than 1" into "less than or equal to 0".
2575 Pred = PPC::getPredicate(Condition: PPC::PRED_LE, Hint: PredHint);
2576 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2577 // We convert "greater than or equal to 1" into "greater than 0".
2578 Pred = PPC::getPredicate(Condition: PPC::PRED_GT, Hint: PredHint);
2579 else
2580 return false;
2581
2582 // Convert the comparison and its user to a compare against zero with the
2583 // appropriate predicate on the branch. Zero comparison might provide
2584 // optimization opportunities post-RA (see optimization in
2585 // PPCPreEmitPeephole.cpp).
2586 UseMI->getOperand(i: 0).setImm(Pred);
2587 CmpInstr.getOperand(i: 2).setImm(0);
2588 }
2589
2590 // Search for Sub.
2591 --I;
2592
2593 // Get ready to iterate backward from CmpInstr.
2594 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2595
2596 for (; I != E && !noSub; --I) {
2597 const MachineInstr &Instr = *I;
2598 unsigned IOpC = Instr.getOpcode();
2599
2600 if (&*I != &CmpInstr && (Instr.modifiesRegister(Reg: PPC::CR0, TRI: &RI) ||
2601 Instr.readsRegister(Reg: PPC::CR0, TRI: &RI)))
2602 // This instruction modifies or uses the record condition register after
2603 // the one we want to change. While we could do this transformation, it
2604 // would likely not be profitable. This transformation removes one
2605 // instruction, and so even forcing RA to generate one move probably
2606 // makes it unprofitable.
2607 return false;
2608
2609 // Check whether CmpInstr can be made redundant by the current instruction.
2610 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2611 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2612 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2613 ((Instr.getOperand(i: 1).getReg() == SrcReg &&
2614 Instr.getOperand(i: 2).getReg() == SrcReg2) ||
2615 (Instr.getOperand(i: 1).getReg() == SrcReg2 &&
2616 Instr.getOperand(i: 2).getReg() == SrcReg))) {
2617 Sub = &*I;
2618 break;
2619 }
2620
2621 if (I == B)
2622 // The 'and' is below the comparison instruction.
2623 return false;
2624 }
2625
2626 // Return false if no candidates exist.
2627 if (!MI && !Sub)
2628 return false;
2629
2630 // The single candidate is called MI.
2631 if (!MI) MI = Sub;
2632
2633 int NewOpC = -1;
2634 int MIOpC = MI->getOpcode();
2635 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2636 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2637 NewOpC = MIOpC;
2638 else {
2639 NewOpC = PPC::getRecordFormOpcode(Opcode: MIOpC);
2640 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(Opcode: MIOpC) != -1)
2641 NewOpC = MIOpC;
2642 }
2643
2644 // FIXME: On the non-embedded POWER architectures, only some of the record
2645 // forms are fast, and we should use only the fast ones.
2646
2647 // The defining instruction has a record form (or is already a record
2648 // form). It is possible, however, that we'll need to reverse the condition
2649 // code of the users.
2650 if (NewOpC == -1)
2651 return false;
2652
2653 // This transformation should not be performed if `nsw` is missing and is not
2654 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2655 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2656 // CRReg can reflect if compared values are equal, this optz is still valid.
2657 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2658 Sub && !Sub->getFlag(Flag: MachineInstr::NoSWrap))
2659 return false;
2660
2661 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2662 // needs to be updated to be based on SUB. Push the condition code
2663 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2664 // condition code of these operands will be modified.
2665 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2666 // comparison against 0, which may modify predicate.
2667 bool ShouldSwap = false;
2668 if (Sub && Value == 0) {
2669 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(i: 1).getReg() == SrcReg2 &&
2670 Sub->getOperand(i: 2).getReg() == SrcReg;
2671
2672 // The operands to subf are the opposite of sub, so only in the fixed-point
2673 // case, invert the order.
2674 ShouldSwap = !ShouldSwap;
2675 }
2676
2677 if (ShouldSwap)
2678 for (MachineRegisterInfo::use_instr_iterator
2679 I = MRI->use_instr_begin(RegNo: CRReg), IE = MRI->use_instr_end();
2680 I != IE; ++I) {
2681 MachineInstr *UseMI = &*I;
2682 if (UseMI->getOpcode() == PPC::BCC) {
2683 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(i: 0).getImm();
2684 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2685 assert((!equalityOnly ||
2686 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2687 "Invalid predicate for equality-only optimization");
2688 (void)PredCond; // To suppress warning in release build.
2689 PredsToUpdate.push_back(Elt: std::make_pair(x: &(UseMI->getOperand(i: 0)),
2690 y: PPC::getSwappedPredicate(Opcode: Pred)));
2691 } else if (UseMI->getOpcode() == PPC::ISEL ||
2692 UseMI->getOpcode() == PPC::ISEL8) {
2693 unsigned NewSubReg = UseMI->getOperand(i: 3).getSubReg();
2694 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2695 "Invalid CR bit for equality-only optimization");
2696
2697 if (NewSubReg == PPC::sub_lt)
2698 NewSubReg = PPC::sub_gt;
2699 else if (NewSubReg == PPC::sub_gt)
2700 NewSubReg = PPC::sub_lt;
2701
2702 SubRegsToUpdate.push_back(Elt: std::make_pair(x: &(UseMI->getOperand(i: 3)),
2703 y&: NewSubReg));
2704 } else // We need to abort on a user we don't understand.
2705 return false;
2706 }
2707 assert(!(Value != 0 && ShouldSwap) &&
2708 "Non-zero immediate support and ShouldSwap"
2709 "may conflict in updating predicate");
2710
2711 // Create a new virtual register to hold the value of the CR set by the
2712 // record-form instruction. If the instruction was not previously in
2713 // record form, then set the kill flag on the CR.
2714 CmpInstr.eraseFromParent();
2715
2716 MachineBasicBlock::iterator MII = MI;
2717 BuildMI(BB&: *MI->getParent(), I: std::next(x: MII), MIMD: MI->getDebugLoc(),
2718 MCID: get(Opcode: TargetOpcode::COPY), DestReg: CRReg)
2719 .addReg(RegNo: PPC::CR0, Flags: getKillRegState(B: MIOpC != NewOpC));
2720
2721 // Even if CR0 register were dead before, it is alive now since the
2722 // instruction we just built uses it.
2723 MI->clearRegisterDeads(Reg: PPC::CR0);
2724
2725 if (MIOpC != NewOpC) {
2726 // We need to be careful here: we're replacing one instruction with
2727 // another, and we need to make sure that we get all of the right
2728 // implicit uses and defs. On the other hand, the caller may be holding
2729 // an iterator to this instruction, and so we can't delete it (this is
2730 // specifically the case if this is the instruction directly after the
2731 // compare).
2732
2733 // Rotates are expensive instructions. If we're emitting a record-form
2734 // rotate that can just be an andi/andis, we should just emit that.
2735 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2736 Register GPRRes = MI->getOperand(i: 0).getReg();
2737 int64_t SH = MI->getOperand(i: 2).getImm();
2738 int64_t MB = MI->getOperand(i: 3).getImm();
2739 int64_t ME = MI->getOperand(i: 4).getImm();
2740 // We can only do this if both the start and end of the mask are in the
2741 // same halfword.
2742 bool MBInLoHWord = MB >= 16;
2743 bool MEInLoHWord = ME >= 16;
2744 uint64_t Mask = ~0LLU;
2745
2746 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2747 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2748 // The mask value needs to shift right 16 if we're emitting andis.
2749 Mask >>= MBInLoHWord ? 0 : 16;
2750 NewOpC = MIOpC == PPC::RLWINM
2751 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2752 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2753 } else if (MRI->use_empty(RegNo: GPRRes) && (ME == 31) &&
2754 (ME - MB + 1 == SH) && (MB >= 16)) {
2755 // If we are rotating by the exact number of bits as are in the mask
2756 // and the mask is in the least significant bits of the register,
2757 // that's just an andis. (as long as the GPR result has no uses).
2758 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2759 Mask >>= 16;
2760 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2761 }
2762 // If we've set the mask, we can transform.
2763 if (Mask != ~0LLU) {
2764 MI->removeOperand(OpNo: 4);
2765 MI->removeOperand(OpNo: 3);
2766 MI->getOperand(i: 2).setImm(Mask);
2767 NumRcRotatesConvertedToRcAnd++;
2768 }
2769 } else if (MIOpC == PPC::RLDICL && MI->getOperand(i: 2).getImm() == 0) {
2770 int64_t MB = MI->getOperand(i: 3).getImm();
2771 if (MB >= 48) {
2772 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2773 NewOpC = PPC::ANDI8_rec;
2774 MI->removeOperand(OpNo: 3);
2775 MI->getOperand(i: 2).setImm(Mask);
2776 NumRcRotatesConvertedToRcAnd++;
2777 }
2778 }
2779
2780 const MCInstrDesc &NewDesc = get(Opcode: NewOpC);
2781 MI->setDesc(NewDesc);
2782
2783 for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2784 if (!MI->definesRegister(Reg: ImpDef, /*TRI=*/nullptr)) {
2785 MI->addOperand(MF&: *MI->getParent()->getParent(),
2786 Op: MachineOperand::CreateReg(Reg: ImpDef, isDef: true, isImp: true));
2787 }
2788 }
2789 for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2790 if (!MI->readsRegister(Reg: ImpUse, /*TRI=*/nullptr)) {
2791 MI->addOperand(MF&: *MI->getParent()->getParent(),
2792 Op: MachineOperand::CreateReg(Reg: ImpUse, isDef: false, isImp: true));
2793 }
2794 }
2795 }
2796 assert(MI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
2797 "Record-form instruction does not define cr0?");
2798
2799 // Modify the condition code of operands in OperandsToUpdate.
2800 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2801 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2802 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2803 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2804
2805 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2806 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2807
2808 return true;
2809}
2810
2811bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
2812 MachineRegisterInfo *MRI = &CmpMI.getParent()->getParent()->getRegInfo();
2813 if (MRI->isSSA())
2814 return false;
2815
2816 Register SrcReg, SrcReg2;
2817 int64_t CmpMask, CmpValue;
2818 if (!analyzeCompare(MI: CmpMI, SrcReg, SrcReg2, Mask&: CmpMask, Value&: CmpValue))
2819 return false;
2820
2821 // Try to optimize the comparison against 0.
2822 if (CmpValue || !CmpMask || SrcReg2)
2823 return false;
2824
2825 // The record forms set the condition register based on a signed comparison
2826 // with zero (see comments in optimizeCompareInstr). Since we can't do the
2827 // equality checks in post-RA, we are more restricted on a unsigned
2828 // comparison.
2829 unsigned Opc = CmpMI.getOpcode();
2830 if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2831 return false;
2832
2833 // The record forms are always based on a 64-bit comparison on PPC64
2834 // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2835 // comparison. Since we can't do the equality checks in post-RA, we bail out
2836 // the case.
2837 if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2838 return false;
2839
2840 // CmpMI can't be deleted if it has implicit def.
2841 if (CmpMI.hasImplicitDef())
2842 return false;
2843
2844 bool SrcRegHasOtherUse = false;
2845 MachineInstr *SrcMI = getDefMIPostRA(Reg: SrcReg, MI&: CmpMI, SeenIntermediateUse&: SrcRegHasOtherUse);
2846 if (!SrcMI || !SrcMI->definesRegister(Reg: SrcReg, /*TRI=*/nullptr))
2847 return false;
2848
2849 MachineOperand RegMO = CmpMI.getOperand(i: 0);
2850 Register CRReg = RegMO.getReg();
2851 if (CRReg != PPC::CR0)
2852 return false;
2853
2854 // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2855 bool SeenUseOfCRReg = false;
2856 bool IsCRRegKilled = false;
2857 if (!isRegElgibleForForwarding(RegMO, DefMI: *SrcMI, MI: CmpMI, KillDefMI: false, IsFwdFeederRegKilled&: IsCRRegKilled,
2858 SeenIntermediateUse&: SeenUseOfCRReg) ||
2859 SrcMI->definesRegister(Reg: CRReg, /*TRI=*/nullptr) || SeenUseOfCRReg)
2860 return false;
2861
2862 int SrcMIOpc = SrcMI->getOpcode();
2863 int NewOpC = PPC::getRecordFormOpcode(Opcode: SrcMIOpc);
2864 if (NewOpC == -1)
2865 return false;
2866
2867 LLVM_DEBUG(dbgs() << "Replace Instr: ");
2868 LLVM_DEBUG(SrcMI->dump());
2869
2870 const MCInstrDesc &NewDesc = get(Opcode: NewOpC);
2871 SrcMI->setDesc(NewDesc);
2872 MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2873 .addReg(RegNo: CRReg, Flags: RegState::ImplicitDefine);
2874 SrcMI->clearRegisterDeads(Reg: CRReg);
2875
2876 assert(SrcMI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
2877 "Record-form instruction does not define cr0?");
2878
2879 LLVM_DEBUG(dbgs() << "with: ");
2880 LLVM_DEBUG(SrcMI->dump());
2881 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2882 LLVM_DEBUG(CmpMI.dump());
2883 return true;
2884}
2885
2886bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
2887 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2888 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
2889 const TargetRegisterInfo *TRI) const {
2890 const MachineOperand *BaseOp;
2891 OffsetIsScalable = false;
2892 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2893 return false;
2894 BaseOps.push_back(Elt: BaseOp);
2895 return true;
2896}
2897
2898static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2899 const TargetRegisterInfo *TRI) {
2900 // If this is a volatile load/store, don't mess with it.
2901 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2902 return false;
2903
2904 if (LdSt.getOperand(i: 2).isFI())
2905 return true;
2906
2907 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2908 // Can't cluster if the instruction modifies the base register
2909 // or it is update form. e.g. ld r2,3(r2)
2910 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 2).getReg(), TRI))
2911 return false;
2912
2913 return true;
2914}
2915
2916// Only cluster instruction pair that have the same opcode, and they are
2917// clusterable according to PowerPC specification.
2918static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2919 const PPCSubtarget &Subtarget) {
2920 switch (FirstOpc) {
2921 default:
2922 return false;
2923 case PPC::STD:
2924 case PPC::STFD:
2925 case PPC::STXSD:
2926 case PPC::DFSTOREf64:
2927 return FirstOpc == SecondOpc;
2928 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2929 // 32bit and 64bit instruction selection. They are clusterable pair though
2930 // they are different opcode.
2931 case PPC::STW:
2932 case PPC::STW8:
2933 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2934 }
2935}
2936
2937bool PPCInstrInfo::shouldClusterMemOps(
2938 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
2939 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2940 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
2941 unsigned NumBytes) const {
2942
2943 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2944 const MachineOperand &BaseOp1 = *BaseOps1.front();
2945 const MachineOperand &BaseOp2 = *BaseOps2.front();
2946 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2947 "Only base registers and frame indices are supported.");
2948
2949 // ClusterSize means the number of memory operations that will have been
2950 // clustered if this hook returns true.
2951 // Don't cluster memory op if there are already two ops clustered at least.
2952 if (ClusterSize > 2)
2953 return false;
2954
2955 // Cluster the load/store only when they have the same base
2956 // register or FI.
2957 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2958 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2959 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2960 return false;
2961
2962 // Check if the load/store are clusterable according to the PowerPC
2963 // specification.
2964 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2965 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2966 unsigned FirstOpc = FirstLdSt.getOpcode();
2967 unsigned SecondOpc = SecondLdSt.getOpcode();
2968 // Cluster the load/store only when they have the same opcode, and they are
2969 // clusterable opcode according to PowerPC specification.
2970 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2971 return false;
2972
2973 // Can't cluster load/store that have ordered or volatile memory reference.
2974 if (!isLdStSafeToCluster(LdSt: FirstLdSt, TRI: &RI) ||
2975 !isLdStSafeToCluster(LdSt: SecondLdSt, TRI: &RI))
2976 return false;
2977
2978 int64_t Offset1 = 0, Offset2 = 0;
2979 LocationSize Width1 = LocationSize::precise(Value: 0),
2980 Width2 = LocationSize::precise(Value: 0);
2981 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2982 if (!getMemOperandWithOffsetWidth(LdSt: FirstLdSt, BaseOp&: Base1, Offset&: Offset1, Width&: Width1, TRI: &RI) ||
2983 !getMemOperandWithOffsetWidth(LdSt: SecondLdSt, BaseOp&: Base2, Offset&: Offset2, Width&: Width2, TRI: &RI) ||
2984 Width1 != Width2)
2985 return false;
2986
2987 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2988 "getMemOperandWithOffsetWidth return incorrect base op");
2989 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2990 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2991 return Offset1 + (int64_t)Width1.getValue() == Offset2;
2992}
2993
2994/// GetInstSize - Return the number of bytes of code the specified
2995/// instruction may be. This returns the maximum number of bytes.
2996///
2997unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2998 unsigned Opcode = MI.getOpcode();
2999
3000 switch (Opcode) {
3001 case PPC::INLINEASM:
3002 case PPC::INLINEASM_BR: {
3003 const MachineFunction *MF = MI.getParent()->getParent();
3004 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
3005 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo());
3006 }
3007 case TargetOpcode::STACKMAP: {
3008 StackMapOpers Opers(&MI);
3009 return Opers.getNumPatchBytes();
3010 }
3011 case TargetOpcode::PATCHPOINT: {
3012 PatchPointOpers Opers(&MI);
3013 return Opers.getNumPatchBytes();
3014 }
3015 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
3016 const MachineFunction *MF = MI.getParent()->getParent();
3017 const Function &F = MF->getFunction();
3018 unsigned Num = F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
3019 if (Num || MF->getTarget().getTargetTriple().isOSAIX() ||
3020 !MF->getTarget().getTargetTriple().isLittleEndian())
3021 return Num * 4;
3022 // Size of xray sled.
3023 return 7 * 4;
3024 }
3025 case TargetOpcode::PATCHABLE_RET: {
3026 // Size of xray sled.
3027 unsigned RetOpcode = MI.getOperand(i: 0).getImm();
3028 bool IsConditional = RetOpcode == PPC::BCCLR;
3029 return (8 + IsConditional) * 4;
3030 }
3031 case TargetOpcode::BUNDLE:
3032 return getInstBundleSize(MI);
3033 default:
3034 return get(Opcode).getSize();
3035 }
3036}
3037
3038TargetInstrInfo::InstSizeVerifyMode
3039PPCInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
3040 // FIXME: The size of STACKMAP is currently over-estimated.
3041 return MI.getOpcode() == TargetOpcode::STACKMAP
3042 ? InstSizeVerifyMode::AllowOverEstimate
3043 : InstSizeVerifyMode::ExactSize;
3044}
3045
3046std::pair<unsigned, unsigned>
3047PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3048 // PPC always uses a direct mask.
3049 return std::make_pair(x&: TF, y: 0u);
3050}
3051
3052ArrayRef<std::pair<unsigned, const char *>>
3053PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3054 using namespace PPCII;
3055 static const std::pair<unsigned, const char *> TargetFlags[] = {
3056 {MO_PLT, "ppc-plt"},
3057 {MO_PIC_FLAG, "ppc-pic"},
3058 {MO_PCREL_FLAG, "ppc-pcrel"},
3059 {MO_GOT_FLAG, "ppc-got"},
3060 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
3061 {MO_TLSGD_FLAG, "ppc-tlsgd"},
3062 {MO_TPREL_FLAG, "ppc-tprel"},
3063 {MO_TLSLDM_FLAG, "ppc-tlsldm"},
3064 {MO_TLSLD_FLAG, "ppc-tlsld"},
3065 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
3066 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
3067 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
3068 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"},
3069 {MO_LO, "ppc-lo"},
3070 {MO_HA, "ppc-ha"},
3071 {MO_TPREL_LO, "ppc-tprel-lo"},
3072 {MO_TPREL_HA, "ppc-tprel-ha"},
3073 {MO_DTPREL_LO, "ppc-dtprel-lo"},
3074 {MO_TLSLD_LO, "ppc-tlsld-lo"},
3075 {MO_TOC_LO, "ppc-toc-lo"},
3076 {MO_TLS, "ppc-tls"},
3077 {MO_PIC_HA_FLAG, "ppc-ha-pic"},
3078 {MO_PIC_LO_FLAG, "ppc-lo-pic"},
3079 {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"},
3080 {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"},
3081 {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"},
3082 };
3083 return ArrayRef(TargetFlags);
3084}
3085
3086// Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
3087// The VSX versions have the advantage of a full 64-register target whereas
3088// the FP ones have the advantage of lower latency and higher throughput. So
3089// what we are after is using the faster instructions in low register pressure
3090// situations and using the larger register file in high register pressure
3091// situations.
3092bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
3093 unsigned UpperOpcode, LowerOpcode;
3094 switch (MI.getOpcode()) {
3095 case PPC::DFLOADf32:
3096 UpperOpcode = PPC::LXSSP;
3097 LowerOpcode = PPC::LFS;
3098 break;
3099 case PPC::DFLOADf64:
3100 UpperOpcode = PPC::LXSD;
3101 LowerOpcode = PPC::LFD;
3102 break;
3103 case PPC::DFSTOREf32:
3104 UpperOpcode = PPC::STXSSP;
3105 LowerOpcode = PPC::STFS;
3106 break;
3107 case PPC::DFSTOREf64:
3108 UpperOpcode = PPC::STXSD;
3109 LowerOpcode = PPC::STFD;
3110 break;
3111 case PPC::XFLOADf32:
3112 UpperOpcode = PPC::LXSSPX;
3113 LowerOpcode = PPC::LFSX;
3114 break;
3115 case PPC::XFLOADf64:
3116 UpperOpcode = PPC::LXSDX;
3117 LowerOpcode = PPC::LFDX;
3118 break;
3119 case PPC::XFSTOREf32:
3120 UpperOpcode = PPC::STXSSPX;
3121 LowerOpcode = PPC::STFSX;
3122 break;
3123 case PPC::XFSTOREf64:
3124 UpperOpcode = PPC::STXSDX;
3125 LowerOpcode = PPC::STFDX;
3126 break;
3127 case PPC::LIWAX:
3128 UpperOpcode = PPC::LXSIWAX;
3129 LowerOpcode = PPC::LFIWAX;
3130 break;
3131 case PPC::LIWZX:
3132 UpperOpcode = PPC::LXSIWZX;
3133 LowerOpcode = PPC::LFIWZX;
3134 break;
3135 case PPC::STIWX:
3136 UpperOpcode = PPC::STXSIWX;
3137 LowerOpcode = PPC::STFIWX;
3138 break;
3139 default:
3140 llvm_unreachable("Unknown Operation!");
3141 }
3142
3143 Register TargetReg = MI.getOperand(i: 0).getReg();
3144 unsigned Opcode;
3145 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3146 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3147 Opcode = LowerOpcode;
3148 else
3149 Opcode = UpperOpcode;
3150 MI.setDesc(get(Opcode));
3151 return true;
3152}
3153
3154static bool isAnImmediateOperand(const MachineOperand &MO) {
3155 return MO.isCPI() || MO.isGlobal() || MO.isImm();
3156}
3157
3158bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
3159 auto &MBB = *MI.getParent();
3160 auto DL = MI.getDebugLoc();
3161
3162 switch (MI.getOpcode()) {
3163 case PPC::BUILD_UACC: {
3164 MCRegister ACC = MI.getOperand(i: 0).getReg();
3165 MCRegister UACC = MI.getOperand(i: 1).getReg();
3166 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3167 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3168 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3169 // FIXME: This can easily be improved to look up to the top of the MBB
3170 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3171 // we can just re-target any such XXLOR's to DstVSR + offset.
3172 for (int VecNo = 0; VecNo < 4; VecNo++)
3173 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: DstVSR + VecNo)
3174 .addReg(RegNo: SrcVSR + VecNo)
3175 .addReg(RegNo: SrcVSR + VecNo);
3176 }
3177 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3178 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3179 // with a NOP.
3180 [[fallthrough]];
3181 }
3182 case PPC::KILL_PAIR: {
3183 MI.setDesc(get(Opcode: PPC::UNENCODED_NOP));
3184 MI.removeOperand(OpNo: 1);
3185 MI.removeOperand(OpNo: 0);
3186 return true;
3187 }
3188 case TargetOpcode::LOAD_STACK_GUARD: {
3189 auto M = MBB.getParent()->getFunction().getParent();
3190 assert(
3191 (Subtarget.isTargetLinux() || M->getStackProtectorGuard() == "tls") &&
3192 "Only Linux target or tls mode are expected to contain "
3193 "LOAD_STACK_GUARD");
3194 int64_t Offset;
3195 if (M->getStackProtectorGuard() == "tls")
3196 Offset = M->getStackProtectorGuardOffset();
3197 else
3198 Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3199 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3200 MI.setDesc(get(Opcode: Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3201 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3202 .addImm(Val: Offset)
3203 .addReg(RegNo: Reg);
3204 return true;
3205 }
3206 case PPC::PPCLdFixedAddr: {
3207 assert((Subtarget.getTargetTriple().isOSGlibc() ||
3208 Subtarget.getTargetTriple().isMusl()) &&
3209 "Only targets with Glibc expected to contain PPCLdFixedAddr");
3210 int64_t Offset = 0;
3211 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3212 MI.setDesc(get(Opcode: PPC::LWZ));
3213 uint64_t FAType = MI.getOperand(i: 1).getImm();
3214#undef PPC_LNX_FEATURE
3215#undef PPC_CPU
3216#define PPC_LNX_DEFINE_OFFSETS
3217#include "llvm/TargetParser/PPCTargetParser.def"
3218 bool IsLE = Subtarget.isLittleEndian();
3219 bool Is64 = Subtarget.isPPC64();
3220 if (FAType == PPC_FAWORD_HWCAP) {
3221 if (IsLE)
3222 Offset = Is64 ? PPC_HWCAP_OFFSET_LE64 : PPC_HWCAP_OFFSET_LE32;
3223 else
3224 Offset = Is64 ? PPC_HWCAP_OFFSET_BE64 : PPC_HWCAP_OFFSET_BE32;
3225 } else if (FAType == PPC_FAWORD_HWCAP2) {
3226 if (IsLE)
3227 Offset = Is64 ? PPC_HWCAP2_OFFSET_LE64 : PPC_HWCAP2_OFFSET_LE32;
3228 else
3229 Offset = Is64 ? PPC_HWCAP2_OFFSET_BE64 : PPC_HWCAP2_OFFSET_BE32;
3230 } else if (FAType == PPC_FAWORD_CPUID) {
3231 if (IsLE)
3232 Offset = Is64 ? PPC_CPUID_OFFSET_LE64 : PPC_CPUID_OFFSET_LE32;
3233 else
3234 Offset = Is64 ? PPC_CPUID_OFFSET_BE64 : PPC_CPUID_OFFSET_BE32;
3235 }
3236 assert(Offset && "Do not know the offset for this fixed addr load");
3237 MI.removeOperand(OpNo: 1);
3238 Subtarget.getTargetMachine().setGlibcHWCAPAccess();
3239 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3240 .addImm(Val: Offset)
3241 .addReg(RegNo: Reg);
3242 return true;
3243#define PPC_TGT_PARSER_UNDEF_MACROS
3244#include "llvm/TargetParser/PPCTargetParser.def"
3245#undef PPC_TGT_PARSER_UNDEF_MACROS
3246 }
3247 case PPC::DFLOADf32:
3248 case PPC::DFLOADf64:
3249 case PPC::DFSTOREf32:
3250 case PPC::DFSTOREf64: {
3251 assert(Subtarget.hasP9Vector() &&
3252 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3253 assert(MI.getOperand(2).isReg() &&
3254 isAnImmediateOperand(MI.getOperand(1)) &&
3255 "D-form op must have register and immediate operands");
3256 return expandVSXMemPseudo(MI);
3257 }
3258 case PPC::XFLOADf32:
3259 case PPC::XFSTOREf32:
3260 case PPC::LIWAX:
3261 case PPC::LIWZX:
3262 case PPC::STIWX: {
3263 assert(Subtarget.hasP8Vector() &&
3264 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3265 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3266 "X-form op must have register and register operands");
3267 return expandVSXMemPseudo(MI);
3268 }
3269 case PPC::XFLOADf64:
3270 case PPC::XFSTOREf64: {
3271 assert(Subtarget.hasVSX() &&
3272 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3273 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3274 "X-form op must have register and register operands");
3275 return expandVSXMemPseudo(MI);
3276 }
3277 case PPC::SPILLTOVSR_LD: {
3278 Register TargetReg = MI.getOperand(i: 0).getReg();
3279 if (PPC::VSFRCRegClass.contains(Reg: TargetReg)) {
3280 MI.setDesc(get(Opcode: PPC::DFLOADf64));
3281 return expandPostRAPseudo(MI);
3282 }
3283 else
3284 MI.setDesc(get(Opcode: PPC::LD));
3285 return true;
3286 }
3287 case PPC::SPILLTOVSR_ST: {
3288 Register SrcReg = MI.getOperand(i: 0).getReg();
3289 if (PPC::VSFRCRegClass.contains(Reg: SrcReg)) {
3290 NumStoreSPILLVSRRCAsVec++;
3291 MI.setDesc(get(Opcode: PPC::DFSTOREf64));
3292 return expandPostRAPseudo(MI);
3293 } else {
3294 NumStoreSPILLVSRRCAsGpr++;
3295 MI.setDesc(get(Opcode: PPC::STD));
3296 }
3297 return true;
3298 }
3299 case PPC::SPILLTOVSR_LDX: {
3300 Register TargetReg = MI.getOperand(i: 0).getReg();
3301 if (PPC::VSFRCRegClass.contains(Reg: TargetReg))
3302 MI.setDesc(get(Opcode: PPC::LXSDX));
3303 else
3304 MI.setDesc(get(Opcode: PPC::LDX));
3305 return true;
3306 }
3307 case PPC::SPILLTOVSR_STX: {
3308 Register SrcReg = MI.getOperand(i: 0).getReg();
3309 if (PPC::VSFRCRegClass.contains(Reg: SrcReg)) {
3310 NumStoreSPILLVSRRCAsVec++;
3311 MI.setDesc(get(Opcode: PPC::STXSDX));
3312 } else {
3313 NumStoreSPILLVSRRCAsGpr++;
3314 MI.setDesc(get(Opcode: PPC::STDX));
3315 }
3316 return true;
3317 }
3318
3319 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3320 case PPC::CFENCE:
3321 case PPC::CFENCE8: {
3322 auto Val = MI.getOperand(i: 0).getReg();
3323 unsigned CmpOp = Subtarget.isPPC64() ? PPC::CMPD : PPC::CMPW;
3324 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: CmpOp), DestReg: PPC::CR7).addReg(RegNo: Val).addReg(RegNo: Val);
3325 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::CTRL_DEP))
3326 .addImm(Val: PPC::PRED_NE_MINUS)
3327 .addReg(RegNo: PPC::CR7)
3328 .addImm(Val: 1);
3329 MI.setDesc(get(Opcode: PPC::ISYNC));
3330 MI.removeOperand(OpNo: 0);
3331 return true;
3332 }
3333 case PPC::LWAT_CSNE_PSEUDO:
3334 case PPC::LDAT_CSNE_PSEUDO:
3335 return expandAMOCSNEPseudo(MI);
3336 }
3337 return false;
3338}
3339
3340// Essentially a compile-time implementation of a compare->isel sequence.
3341// It takes two constants to compare, along with the true/false registers
3342// and the comparison type (as a subreg to a CR field) and returns one
3343// of the true/false registers, depending on the comparison results.
3344static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3345 unsigned TrueReg, unsigned FalseReg,
3346 unsigned CRSubReg) {
3347 // Signed comparisons. The immediates are assumed to be sign-extended.
3348 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3349 switch (CRSubReg) {
3350 default: llvm_unreachable("Unknown integer comparison type.");
3351 case PPC::sub_lt:
3352 return Imm1 < Imm2 ? TrueReg : FalseReg;
3353 case PPC::sub_gt:
3354 return Imm1 > Imm2 ? TrueReg : FalseReg;
3355 case PPC::sub_eq:
3356 return Imm1 == Imm2 ? TrueReg : FalseReg;
3357 }
3358 }
3359 // Unsigned comparisons.
3360 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3361 switch (CRSubReg) {
3362 default: llvm_unreachable("Unknown integer comparison type.");
3363 case PPC::sub_lt:
3364 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3365 case PPC::sub_gt:
3366 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3367 case PPC::sub_eq:
3368 return Imm1 == Imm2 ? TrueReg : FalseReg;
3369 }
3370 }
3371 return PPC::NoRegister;
3372}
3373
3374void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
3375 unsigned OpNo,
3376 int64_t Imm) const {
3377 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3378 // Replace the REG with the Immediate.
3379 Register InUseReg = MI.getOperand(i: OpNo).getReg();
3380 MI.getOperand(i: OpNo).ChangeToImmediate(ImmVal: Imm);
3381
3382 // We need to make sure that the MI didn't have any implicit use
3383 // of this REG any more. We don't call MI.implicit_operands().empty() to
3384 // return early, since MI's MCID might be changed in calling context, as a
3385 // result its number of explicit operands may be changed, thus the begin of
3386 // implicit operand is changed.
3387 int UseOpIdx = MI.findRegisterUseOperandIdx(Reg: InUseReg, TRI: &RI, isKill: false);
3388 if (UseOpIdx >= 0) {
3389 MachineOperand &MO = MI.getOperand(i: UseOpIdx);
3390 if (MO.isImplicit())
3391 // The operands must always be in the following order:
3392 // - explicit reg defs,
3393 // - other explicit operands (reg uses, immediates, etc.),
3394 // - implicit reg defs
3395 // - implicit reg uses
3396 // Therefore, removing the implicit operand won't change the explicit
3397 // operands layout.
3398 MI.removeOperand(OpNo: UseOpIdx);
3399 }
3400}
3401
3402// Replace an instruction with one that materializes a constant (and sets
3403// CR0 if the original instruction was a record-form instruction).
3404void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
3405 const LoadImmediateInfo &LII) const {
3406 // Remove existing operands.
3407 int OperandToKeep = LII.SetCR ? 1 : 0;
3408 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3409 MI.removeOperand(OpNo: i);
3410
3411 // Replace the instruction.
3412 if (LII.SetCR) {
3413 MI.setDesc(get(Opcode: LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3414 // Set the immediate.
3415 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3416 .addImm(Val: LII.Imm).addReg(RegNo: PPC::CR0, Flags: RegState::ImplicitDefine);
3417 return;
3418 }
3419 else
3420 MI.setDesc(get(Opcode: LII.Is64Bit ? PPC::LI8 : PPC::LI));
3421
3422 // Set the immediate.
3423 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3424 .addImm(Val: LII.Imm);
3425}
3426
3427MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
3428 bool &SeenIntermediateUse) const {
3429 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3430 "Should be called after register allocation.");
3431 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3432 It++;
3433 SeenIntermediateUse = false;
3434 for (; It != E; ++It) {
3435 if (It->modifiesRegister(Reg, TRI: &RI))
3436 return &*It;
3437 if (It->readsRegister(Reg, TRI: &RI))
3438 SeenIntermediateUse = true;
3439 }
3440 return nullptr;
3441}
3442
3443void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB,
3444 MachineBasicBlock::iterator MBBI,
3445 const DebugLoc &DL, Register Reg,
3446 int64_t Imm) const {
3447 assert(!MBB.getParent()->getRegInfo().isSSA() &&
3448 "Register should be in non-SSA form after RA");
3449 bool isPPC64 = Subtarget.isPPC64();
3450 // FIXME: Materialization here is not optimal.
3451 // For some special bit patterns we can use less instructions.
3452 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3453 if (isInt<16>(x: Imm)) {
3454 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: Reg).addImm(Val: Imm);
3455 } else if (isInt<32>(x: Imm)) {
3456 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: Reg)
3457 .addImm(Val: Imm >> 16);
3458 if (Imm & 0xFFFF)
3459 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI), DestReg: Reg)
3460 .addReg(RegNo: Reg, Flags: RegState::Kill)
3461 .addImm(Val: Imm & 0xFFFF);
3462 } else {
3463 assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3464 "only supported in PPC64");
3465 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::LIS8), DestReg: Reg).addImm(Val: Imm >> 48);
3466 if ((Imm >> 32) & 0xFFFF)
3467 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORI8), DestReg: Reg)
3468 .addReg(RegNo: Reg, Flags: RegState::Kill)
3469 .addImm(Val: (Imm >> 32) & 0xFFFF);
3470 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::RLDICR), DestReg: Reg)
3471 .addReg(RegNo: Reg, Flags: RegState::Kill)
3472 .addImm(Val: 32)
3473 .addImm(Val: 31);
3474 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORIS8), DestReg: Reg)
3475 .addReg(RegNo: Reg, Flags: RegState::Kill)
3476 .addImm(Val: (Imm >> 16) & 0xFFFF);
3477 if (Imm & 0xFFFF)
3478 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORI8), DestReg: Reg)
3479 .addReg(RegNo: Reg, Flags: RegState::Kill)
3480 .addImm(Val: Imm & 0xFFFF);
3481 }
3482}
3483
3484MachineInstr *PPCInstrInfo::getForwardingDefMI(
3485 MachineInstr &MI,
3486 unsigned &OpNoForForwarding,
3487 bool &SeenIntermediateUse) const {
3488 OpNoForForwarding = ~0U;
3489 MachineInstr *DefMI = nullptr;
3490 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3491 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3492 // within the basic block to see if the register is defined using an
3493 // LI/LI8/ADDI/ADDI8.
3494 if (MRI->isSSA()) {
3495 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3496 if (!MI.getOperand(i).isReg())
3497 continue;
3498 Register Reg = MI.getOperand(i).getReg();
3499 if (!Reg.isVirtual())
3500 continue;
3501 Register TrueReg = RI.lookThruCopyLike(SrcReg: Reg, MRI);
3502 if (TrueReg.isVirtual()) {
3503 MachineInstr *DefMIForTrueReg = MRI->getVRegDef(Reg: TrueReg);
3504 if (!DefMIForTrueReg)
3505 continue;
3506 if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3507 DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3508 DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3509 DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3510 OpNoForForwarding = i;
3511 DefMI = DefMIForTrueReg;
3512 // The ADDI and LI operand maybe exist in one instruction at same
3513 // time. we prefer to fold LI operand as LI only has one Imm operand
3514 // and is more possible to be converted. So if current DefMI is
3515 // ADDI/ADDI8, we continue to find possible LI/LI8.
3516 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3517 break;
3518 }
3519 }
3520 }
3521 } else {
3522 // Looking back through the definition for each operand could be expensive,
3523 // so exit early if this isn't an instruction that either has an immediate
3524 // form or is already an immediate form that we can handle.
3525 ImmInstrInfo III;
3526 unsigned Opc = MI.getOpcode();
3527 bool ConvertibleImmForm =
3528 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3529 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3530 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3531 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3532 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3533 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3534 Opc == PPC::RLWINM8_rec;
3535 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(i: 0).isReg())
3536 ? PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg())
3537 : false;
3538 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, PostRA: true))
3539 return nullptr;
3540
3541 // Don't convert or %X, %Y, %Y since that's just a register move.
3542 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3543 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
3544 return nullptr;
3545 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3546 MachineOperand &MO = MI.getOperand(i);
3547 SeenIntermediateUse = false;
3548 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3549 Register Reg = MI.getOperand(i).getReg();
3550 // If we see another use of this reg between the def and the MI,
3551 // we want to flag it so the def isn't deleted.
3552 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3553 if (DefMI) {
3554 // Is this register defined by some form of add-immediate (including
3555 // load-immediate) within this basic block?
3556 switch (DefMI->getOpcode()) {
3557 default:
3558 break;
3559 case PPC::LI:
3560 case PPC::LI8:
3561 case PPC::ADDItocL8:
3562 case PPC::ADDI:
3563 case PPC::ADDI8:
3564 OpNoForForwarding = i;
3565 return DefMI;
3566 }
3567 }
3568 }
3569 }
3570 }
3571 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3572}
3573
3574unsigned PPCInstrInfo::getSpillTarget() const {
3575 // With P10, we may need to spill paired vector registers or accumulator
3576 // registers. MMA implies paired vectors, so we can just check that.
3577 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3578 // P11 uses the P10 target.
3579 return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3580 2 : Subtarget.hasP9Vector() ?
3581 1 : 0;
3582}
3583
3584ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3585 return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3586}
3587
3588ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3589 return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3590}
3591
3592// This opt tries to convert the following imm form to an index form to save an
3593// add for stack variables.
3594// Return false if no such pattern found.
3595//
3596// ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3597// ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3598// Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3599//
3600// can be converted to:
3601//
3602// new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3603// Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3604//
3605// In order to eliminate ADD instr, make sure that:
3606// 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3607// new ADDI instr and ADDI can only take int16 Imm.
3608// 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3609// between ADDI and ADD instr since its original def in ADDI will be changed
3610// in new ADDI instr. And also there should be no new def for it between
3611// ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3612// 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3613// between ADD and Imm instr since ADD instr will be eliminated.
3614// 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3615// moved to Index instr.
3616bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
3617 MachineFunction *MF = MI.getParent()->getParent();
3618 MachineRegisterInfo *MRI = &MF->getRegInfo();
3619 bool PostRA = !MRI->isSSA();
3620 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3621 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3622 // frame base(OffsetAddi) are determined.
3623 if (!PostRA)
3624 return false;
3625 unsigned ToBeDeletedReg = 0;
3626 int64_t OffsetImm = 0;
3627 unsigned XFormOpcode = 0;
3628 ImmInstrInfo III;
3629
3630 // Check if Imm instr meets requirement.
3631 if (!isImmInstrEligibleForFolding(MI, BaseReg&: ToBeDeletedReg, XFormOpcode, OffsetOfImmInstr&: OffsetImm,
3632 III))
3633 return false;
3634
3635 bool OtherIntermediateUse = false;
3636 MachineInstr *ADDMI = getDefMIPostRA(Reg: ToBeDeletedReg, MI, SeenIntermediateUse&: OtherIntermediateUse);
3637
3638 // Exit if there is other use between ADD and Imm instr or no def found.
3639 if (OtherIntermediateUse || !ADDMI)
3640 return false;
3641
3642 // Check if ADD instr meets requirement.
3643 if (!isADDInstrEligibleForFolding(ADDMI&: *ADDMI))
3644 return false;
3645
3646 unsigned ScaleRegIdx = 0;
3647 int64_t OffsetAddi = 0;
3648 MachineInstr *ADDIMI = nullptr;
3649
3650 // Check if there is a valid ToBeChangedReg in ADDMI.
3651 // 1: It must be killed.
3652 // 2: Its definition must be a valid ADDIMI.
3653 // 3: It must satify int16 offset requirement.
3654 if (isValidToBeChangedReg(ADDMI, Index: 1, ADDIMI, OffsetAddi, OffsetImm))
3655 ScaleRegIdx = 2;
3656 else if (isValidToBeChangedReg(ADDMI, Index: 2, ADDIMI, OffsetAddi, OffsetImm))
3657 ScaleRegIdx = 1;
3658 else
3659 return false;
3660
3661 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3662 Register ToBeChangedReg = ADDIMI->getOperand(i: 0).getReg();
3663 Register ScaleReg = ADDMI->getOperand(i: ScaleRegIdx).getReg();
3664 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3665 MachineBasicBlock::iterator End) {
3666 for (auto It = ++Start; It != End; It++)
3667 if (It->modifiesRegister(Reg, TRI: &getRegisterInfo()))
3668 return true;
3669 return false;
3670 };
3671
3672 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3673 // treated as special zero when ScaleReg is R0/X0 register.
3674 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3675 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3676 return false;
3677
3678 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3679 // and Imm Instr.
3680 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3681 return false;
3682
3683 // Now start to do the transformation.
3684 LLVM_DEBUG(dbgs() << "Replace instruction: "
3685 << "\n");
3686 LLVM_DEBUG(ADDIMI->dump());
3687 LLVM_DEBUG(ADDMI->dump());
3688 LLVM_DEBUG(MI.dump());
3689 LLVM_DEBUG(dbgs() << "with: "
3690 << "\n");
3691
3692 // Update ADDI instr.
3693 ADDIMI->getOperand(i: 2).setImm(OffsetAddi + OffsetImm);
3694
3695 // Update Imm instr.
3696 MI.setDesc(get(Opcode: XFormOpcode));
3697 MI.getOperand(i: III.ImmOpNo)
3698 .ChangeToRegister(Reg: ScaleReg, isDef: false, isImp: false,
3699 isKill: ADDMI->getOperand(i: ScaleRegIdx).isKill());
3700
3701 MI.getOperand(i: III.OpNoForForwarding)
3702 .ChangeToRegister(Reg: ToBeChangedReg, isDef: false, isImp: false, isKill: true);
3703
3704 // Eliminate ADD instr.
3705 ADDMI->eraseFromParent();
3706
3707 LLVM_DEBUG(ADDIMI->dump());
3708 LLVM_DEBUG(MI.dump());
3709
3710 return true;
3711}
3712
3713bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
3714 int64_t &Imm) const {
3715 unsigned Opc = ADDIMI.getOpcode();
3716
3717 // Exit if the instruction is not ADDI.
3718 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3719 return false;
3720
3721 // The operand may not necessarily be an immediate - it could be a relocation.
3722 if (!ADDIMI.getOperand(i: 2).isImm())
3723 return false;
3724
3725 Imm = ADDIMI.getOperand(i: 2).getImm();
3726
3727 return true;
3728}
3729
3730bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
3731 unsigned Opc = ADDMI.getOpcode();
3732
3733 // Exit if the instruction is not ADD.
3734 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3735}
3736
3737bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
3738 unsigned &ToBeDeletedReg,
3739 unsigned &XFormOpcode,
3740 int64_t &OffsetImm,
3741 ImmInstrInfo &III) const {
3742 // Only handle load/store.
3743 if (!MI.mayLoadOrStore())
3744 return false;
3745
3746 unsigned Opc = MI.getOpcode();
3747
3748 XFormOpcode = RI.getMappedIdxOpcForImmOpc(ImmOpcode: Opc);
3749
3750 // Exit if instruction has no index form.
3751 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3752 return false;
3753
3754 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3755 if (!instrHasImmForm(Opc: XFormOpcode,
3756 IsVFReg: PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg()), III, PostRA: true))
3757 return false;
3758
3759 if (!III.IsSummingOperands)
3760 return false;
3761
3762 MachineOperand ImmOperand = MI.getOperand(i: III.ImmOpNo);
3763 MachineOperand RegOperand = MI.getOperand(i: III.OpNoForForwarding);
3764 // Only support imm operands, not relocation slots or others.
3765 if (!ImmOperand.isImm())
3766 return false;
3767
3768 assert(RegOperand.isReg() && "Instruction format is not right");
3769
3770 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3771 if (!RegOperand.isKill())
3772 return false;
3773
3774 ToBeDeletedReg = RegOperand.getReg();
3775 OffsetImm = ImmOperand.getImm();
3776
3777 return true;
3778}
3779
3780bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
3781 MachineInstr *&ADDIMI,
3782 int64_t &OffsetAddi,
3783 int64_t OffsetImm) const {
3784 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3785 MachineOperand &MO = ADDMI->getOperand(i: Index);
3786
3787 if (!MO.isKill())
3788 return false;
3789
3790 bool OtherIntermediateUse = false;
3791
3792 ADDIMI = getDefMIPostRA(Reg: MO.getReg(), MI&: *ADDMI, SeenIntermediateUse&: OtherIntermediateUse);
3793 // Currently handle only one "add + Imminstr" pair case, exit if other
3794 // intermediate use for ToBeChangedReg found.
3795 // TODO: handle the cases where there are other "add + Imminstr" pairs
3796 // with same offset in Imminstr which is like:
3797 //
3798 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3799 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3800 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3801 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3802 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3803 //
3804 // can be converted to:
3805 //
3806 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3807 // (OffsetAddi + OffsetImm)
3808 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3809 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3810
3811 if (OtherIntermediateUse || !ADDIMI)
3812 return false;
3813 // Check if ADDI instr meets requirement.
3814 if (!isADDIInstrEligibleForFolding(ADDIMI&: *ADDIMI, Imm&: OffsetAddi))
3815 return false;
3816
3817 if (isInt<16>(x: OffsetAddi + OffsetImm))
3818 return true;
3819 return false;
3820}
3821
3822// If this instruction has an immediate form and one of its operands is a
3823// result of a load-immediate or an add-immediate, convert it to
3824// the immediate form if the constant is in range.
3825bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
3826 SmallSet<Register, 4> &RegsToUpdate,
3827 MachineInstr **KilledDef) const {
3828 MachineFunction *MF = MI.getParent()->getParent();
3829 MachineRegisterInfo *MRI = &MF->getRegInfo();
3830 bool PostRA = !MRI->isSSA();
3831 bool SeenIntermediateUse = true;
3832 unsigned ForwardingOperand = ~0U;
3833 MachineInstr *DefMI = getForwardingDefMI(MI, OpNoForForwarding&: ForwardingOperand,
3834 SeenIntermediateUse);
3835 if (!DefMI)
3836 return false;
3837 assert(ForwardingOperand < MI.getNumOperands() &&
3838 "The forwarding operand needs to be valid at this point");
3839 bool IsForwardingOperandKilled = MI.getOperand(i: ForwardingOperand).isKill();
3840 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3841 if (KilledDef && KillFwdDefMI)
3842 *KilledDef = DefMI;
3843
3844 // Conservatively add defs from DefMI and defs/uses from MI to the set of
3845 // registers that need their kill flags updated.
3846 for (const MachineOperand &MO : DefMI->operands())
3847 if (MO.isReg() && MO.isDef())
3848 RegsToUpdate.insert(V: MO.getReg());
3849 for (const MachineOperand &MO : MI.operands())
3850 if (MO.isReg())
3851 RegsToUpdate.insert(V: MO.getReg());
3852
3853 // If this is a imm instruction and its register operands is produced by ADDI,
3854 // put the imm into imm inst directly.
3855 if (RI.getMappedIdxOpcForImmOpc(ImmOpcode: MI.getOpcode()) !=
3856 PPC::INSTRUCTION_LIST_END &&
3857 transformToNewImmFormFedByAdd(MI, DefMI&: *DefMI, OpNoForForwarding: ForwardingOperand))
3858 return true;
3859
3860 ImmInstrInfo III;
3861 bool IsVFReg = MI.getOperand(i: 0).isReg() &&
3862 MI.getOperand(i: 0).getReg().isPhysical() &&
3863 PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg());
3864 bool HasImmForm = instrHasImmForm(Opc: MI.getOpcode(), IsVFReg, III, PostRA);
3865 // If this is a reg+reg instruction that has a reg+imm form,
3866 // and one of the operands is produced by an add-immediate,
3867 // try to convert it.
3868 if (HasImmForm &&
3869 transformToImmFormFedByAdd(MI, III, ConstantOpNo: ForwardingOperand, DefMI&: *DefMI,
3870 KillDefMI: KillFwdDefMI))
3871 return true;
3872
3873 // If this is a reg+reg instruction that has a reg+imm form,
3874 // and one of the operands is produced by LI, convert it now.
3875 if (HasImmForm &&
3876 transformToImmFormFedByLI(MI, III, ConstantOpNo: ForwardingOperand, DefMI&: *DefMI))
3877 return true;
3878
3879 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3880 // can be simpified to LI.
3881 if (!HasImmForm &&
3882 simplifyToLI(MI, DefMI&: *DefMI, OpNoForForwarding: ForwardingOperand, KilledDef, RegsToUpdate: &RegsToUpdate))
3883 return true;
3884
3885 return false;
3886}
3887
3888bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
3889 MachineInstr **ToErase) const {
3890 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3891 Register FoldingReg = MI.getOperand(i: 1).getReg();
3892 if (!FoldingReg.isVirtual())
3893 return false;
3894 MachineInstr *SrcMI = MRI->getVRegDef(Reg: FoldingReg);
3895 if (!SrcMI)
3896 return false;
3897 if (SrcMI->getOpcode() != PPC::RLWINM &&
3898 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3899 SrcMI->getOpcode() != PPC::RLWINM8 &&
3900 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3901 return false;
3902 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3903 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3904 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3905 "Invalid PPC::RLWINM Instruction!");
3906 uint64_t SHSrc = SrcMI->getOperand(i: 2).getImm();
3907 uint64_t SHMI = MI.getOperand(i: 2).getImm();
3908 uint64_t MBSrc = SrcMI->getOperand(i: 3).getImm();
3909 uint64_t MBMI = MI.getOperand(i: 3).getImm();
3910 uint64_t MESrc = SrcMI->getOperand(i: 4).getImm();
3911 uint64_t MEMI = MI.getOperand(i: 4).getImm();
3912
3913 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3914 "Invalid PPC::RLWINM Instruction!");
3915 // If MBMI is bigger than MEMI, we always can not get run of ones.
3916 // RotatedSrcMask non-wrap:
3917 // 0........31|32........63
3918 // RotatedSrcMask: B---E B---E
3919 // MaskMI: -----------|--E B------
3920 // Result: ----- --- (Bad candidate)
3921 //
3922 // RotatedSrcMask wrap:
3923 // 0........31|32........63
3924 // RotatedSrcMask: --E B----|--E B----
3925 // MaskMI: -----------|--E B------
3926 // Result: --- -----|--- ----- (Bad candidate)
3927 //
3928 // One special case is RotatedSrcMask is a full set mask.
3929 // RotatedSrcMask full:
3930 // 0........31|32........63
3931 // RotatedSrcMask: ------EB---|-------EB---
3932 // MaskMI: -----------|--E B------
3933 // Result: -----------|--- ------- (Good candidate)
3934
3935 // Mark special case.
3936 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3937
3938 // For other MBMI > MEMI cases, just return.
3939 if ((MBMI > MEMI) && !SrcMaskFull)
3940 return false;
3941
3942 // Handle MBMI <= MEMI cases.
3943 APInt MaskMI = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - MEMI - 1, hiBit: 32 - MBMI);
3944 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3945 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3946 // while in PowerPC ISA, lowerest bit is at index 63.
3947 APInt MaskSrc = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - MESrc - 1, hiBit: 32 - MBSrc);
3948
3949 APInt RotatedSrcMask = MaskSrc.rotl(rotateAmt: SHMI);
3950 APInt FinalMask = RotatedSrcMask & MaskMI;
3951 uint32_t NewMB, NewME;
3952 bool Simplified = false;
3953
3954 // If final mask is 0, MI result should be 0 too.
3955 if (FinalMask.isZero()) {
3956 bool Is64Bit =
3957 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3958 Simplified = true;
3959 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3960 LLVM_DEBUG(MI.dump());
3961
3962 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3963 // Replace MI with "LI 0"
3964 MI.removeOperand(OpNo: 4);
3965 MI.removeOperand(OpNo: 3);
3966 MI.removeOperand(OpNo: 2);
3967 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: 0);
3968 MI.setDesc(get(Opcode: Is64Bit ? PPC::LI8 : PPC::LI));
3969 } else {
3970 // Replace MI with "ANDI_rec reg, 0"
3971 MI.removeOperand(OpNo: 4);
3972 MI.removeOperand(OpNo: 3);
3973 MI.getOperand(i: 2).setImm(0);
3974 MI.setDesc(get(Opcode: Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3975 MI.getOperand(i: 1).setReg(SrcMI->getOperand(i: 1).getReg());
3976 if (SrcMI->getOperand(i: 1).isKill()) {
3977 MI.getOperand(i: 1).setIsKill(true);
3978 SrcMI->getOperand(i: 1).setIsKill(false);
3979 } else
3980 // About to replace MI.getOperand(1), clear its kill flag.
3981 MI.getOperand(i: 1).setIsKill(false);
3982 }
3983
3984 LLVM_DEBUG(dbgs() << "With: ");
3985 LLVM_DEBUG(MI.dump());
3986
3987 } else if ((isRunOfOnes(Val: (unsigned)(FinalMask.getZExtValue()), MB&: NewMB, ME&: NewME) &&
3988 NewMB <= NewME) ||
3989 SrcMaskFull) {
3990 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3991 // than NewME. Otherwise we get a 64 bit value after folding, but MI
3992 // return a 32 bit value.
3993 Simplified = true;
3994 LLVM_DEBUG(dbgs() << "Converting Instr: ");
3995 LLVM_DEBUG(MI.dump());
3996
3997 uint16_t NewSH = (SHSrc + SHMI) % 32;
3998 MI.getOperand(i: 2).setImm(NewSH);
3999 // If SrcMI mask is full, no need to update MBMI and MEMI.
4000 if (!SrcMaskFull) {
4001 MI.getOperand(i: 3).setImm(NewMB);
4002 MI.getOperand(i: 4).setImm(NewME);
4003 }
4004 MI.getOperand(i: 1).setReg(SrcMI->getOperand(i: 1).getReg());
4005 if (SrcMI->getOperand(i: 1).isKill()) {
4006 MI.getOperand(i: 1).setIsKill(true);
4007 SrcMI->getOperand(i: 1).setIsKill(false);
4008 } else
4009 // About to replace MI.getOperand(1), clear its kill flag.
4010 MI.getOperand(i: 1).setIsKill(false);
4011
4012 LLVM_DEBUG(dbgs() << "To: ");
4013 LLVM_DEBUG(MI.dump());
4014 }
4015 if (Simplified & MRI->use_nodbg_empty(RegNo: FoldingReg) &&
4016 !SrcMI->hasImplicitDef()) {
4017 // If FoldingReg has no non-debug use and it has no implicit def (it
4018 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
4019 // Otherwise keep it.
4020 *ToErase = SrcMI;
4021 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
4022 LLVM_DEBUG(SrcMI->dump());
4023 }
4024 return Simplified;
4025}
4026
4027bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
4028 ImmInstrInfo &III, bool PostRA) const {
4029 // The vast majority of the instructions would need their operand 2 replaced
4030 // with an immediate when switching to the reg+imm form. A marked exception
4031 // are the update form loads/stores for which a constant operand 2 would need
4032 // to turn into a displacement and move operand 1 to the operand 2 position.
4033 III.ImmOpNo = 2;
4034 III.OpNoForForwarding = 2;
4035 III.ImmWidth = 16;
4036 III.ImmMustBeMultipleOf = 1;
4037 III.TruncateImmTo = 0;
4038 III.IsSummingOperands = false;
4039 switch (Opc) {
4040 default: return false;
4041 case PPC::ADD4:
4042 case PPC::ADD8:
4043 III.SignedImm = true;
4044 III.ZeroIsSpecialOrig = 0;
4045 III.ZeroIsSpecialNew = 1;
4046 III.IsCommutative = true;
4047 III.IsSummingOperands = true;
4048 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
4049 break;
4050 case PPC::ADDC:
4051 case PPC::ADDC8:
4052 III.SignedImm = true;
4053 III.ZeroIsSpecialOrig = 0;
4054 III.ZeroIsSpecialNew = 0;
4055 III.IsCommutative = true;
4056 III.IsSummingOperands = true;
4057 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
4058 break;
4059 case PPC::ADDC_rec:
4060 III.SignedImm = true;
4061 III.ZeroIsSpecialOrig = 0;
4062 III.ZeroIsSpecialNew = 0;
4063 III.IsCommutative = true;
4064 III.IsSummingOperands = true;
4065 III.ImmOpcode = PPC::ADDIC_rec;
4066 break;
4067 case PPC::SUBFC:
4068 case PPC::SUBFC8:
4069 III.SignedImm = true;
4070 III.ZeroIsSpecialOrig = 0;
4071 III.ZeroIsSpecialNew = 0;
4072 III.IsCommutative = false;
4073 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
4074 break;
4075 case PPC::CMPW:
4076 case PPC::CMPD:
4077 III.SignedImm = true;
4078 III.ZeroIsSpecialOrig = 0;
4079 III.ZeroIsSpecialNew = 0;
4080 III.IsCommutative = false;
4081 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
4082 break;
4083 case PPC::CMPLW:
4084 case PPC::CMPLD:
4085 III.SignedImm = false;
4086 III.ZeroIsSpecialOrig = 0;
4087 III.ZeroIsSpecialNew = 0;
4088 III.IsCommutative = false;
4089 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
4090 break;
4091 case PPC::AND_rec:
4092 case PPC::AND8_rec:
4093 case PPC::OR:
4094 case PPC::OR8:
4095 case PPC::XOR:
4096 case PPC::XOR8:
4097 III.SignedImm = false;
4098 III.ZeroIsSpecialOrig = 0;
4099 III.ZeroIsSpecialNew = 0;
4100 III.IsCommutative = true;
4101 switch(Opc) {
4102 default: llvm_unreachable("Unknown opcode");
4103 case PPC::AND_rec:
4104 III.ImmOpcode = PPC::ANDI_rec;
4105 break;
4106 case PPC::AND8_rec:
4107 III.ImmOpcode = PPC::ANDI8_rec;
4108 break;
4109 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
4110 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
4111 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
4112 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
4113 }
4114 break;
4115 case PPC::RLWNM:
4116 case PPC::RLWNM8:
4117 case PPC::RLWNM_rec:
4118 case PPC::RLWNM8_rec:
4119 case PPC::SLW:
4120 case PPC::SLW8:
4121 case PPC::SLW_rec:
4122 case PPC::SLW8_rec:
4123 case PPC::SRW:
4124 case PPC::SRW8:
4125 case PPC::SRW_rec:
4126 case PPC::SRW8_rec:
4127 case PPC::SRAW:
4128 case PPC::SRAW_rec:
4129 III.SignedImm = false;
4130 III.ZeroIsSpecialOrig = 0;
4131 III.ZeroIsSpecialNew = 0;
4132 III.IsCommutative = false;
4133 // This isn't actually true, but the instructions ignore any of the
4134 // upper bits, so any immediate loaded with an LI is acceptable.
4135 // This does not apply to shift right algebraic because a value
4136 // out of range will produce a -1/0.
4137 III.ImmWidth = 16;
4138 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4139 Opc == PPC::RLWNM8_rec)
4140 III.TruncateImmTo = 5;
4141 else
4142 III.TruncateImmTo = 6;
4143 switch(Opc) {
4144 default: llvm_unreachable("Unknown opcode");
4145 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4146 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4147 case PPC::RLWNM_rec:
4148 III.ImmOpcode = PPC::RLWINM_rec;
4149 break;
4150 case PPC::RLWNM8_rec:
4151 III.ImmOpcode = PPC::RLWINM8_rec;
4152 break;
4153 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4154 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4155 case PPC::SLW_rec:
4156 III.ImmOpcode = PPC::RLWINM_rec;
4157 break;
4158 case PPC::SLW8_rec:
4159 III.ImmOpcode = PPC::RLWINM8_rec;
4160 break;
4161 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4162 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4163 case PPC::SRW_rec:
4164 III.ImmOpcode = PPC::RLWINM_rec;
4165 break;
4166 case PPC::SRW8_rec:
4167 III.ImmOpcode = PPC::RLWINM8_rec;
4168 break;
4169 case PPC::SRAW:
4170 III.ImmWidth = 5;
4171 III.TruncateImmTo = 0;
4172 III.ImmOpcode = PPC::SRAWI;
4173 break;
4174 case PPC::SRAW_rec:
4175 III.ImmWidth = 5;
4176 III.TruncateImmTo = 0;
4177 III.ImmOpcode = PPC::SRAWI_rec;
4178 break;
4179 }
4180 break;
4181 case PPC::RLDCL:
4182 case PPC::RLDCL_rec:
4183 case PPC::RLDCR:
4184 case PPC::RLDCR_rec:
4185 case PPC::SLD:
4186 case PPC::SLD_rec:
4187 case PPC::SRD:
4188 case PPC::SRD_rec:
4189 case PPC::SRAD:
4190 case PPC::SRAD_rec:
4191 III.SignedImm = false;
4192 III.ZeroIsSpecialOrig = 0;
4193 III.ZeroIsSpecialNew = 0;
4194 III.IsCommutative = false;
4195 // This isn't actually true, but the instructions ignore any of the
4196 // upper bits, so any immediate loaded with an LI is acceptable.
4197 // This does not apply to shift right algebraic because a value
4198 // out of range will produce a -1/0.
4199 III.ImmWidth = 16;
4200 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4201 Opc == PPC::RLDCR_rec)
4202 III.TruncateImmTo = 6;
4203 else
4204 III.TruncateImmTo = 7;
4205 switch(Opc) {
4206 default: llvm_unreachable("Unknown opcode");
4207 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4208 case PPC::RLDCL_rec:
4209 III.ImmOpcode = PPC::RLDICL_rec;
4210 break;
4211 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4212 case PPC::RLDCR_rec:
4213 III.ImmOpcode = PPC::RLDICR_rec;
4214 break;
4215 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4216 case PPC::SLD_rec:
4217 III.ImmOpcode = PPC::RLDICR_rec;
4218 break;
4219 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4220 case PPC::SRD_rec:
4221 III.ImmOpcode = PPC::RLDICL_rec;
4222 break;
4223 case PPC::SRAD:
4224 III.ImmWidth = 6;
4225 III.TruncateImmTo = 0;
4226 III.ImmOpcode = PPC::SRADI;
4227 break;
4228 case PPC::SRAD_rec:
4229 III.ImmWidth = 6;
4230 III.TruncateImmTo = 0;
4231 III.ImmOpcode = PPC::SRADI_rec;
4232 break;
4233 }
4234 break;
4235 // Loads and stores:
4236 case PPC::LBZX:
4237 case PPC::LBZX8:
4238 case PPC::LHZX:
4239 case PPC::LHZX8:
4240 case PPC::LHAX:
4241 case PPC::LHAX8:
4242 case PPC::LWZX:
4243 case PPC::LWZX8:
4244 case PPC::LWAX:
4245 case PPC::LDX:
4246 case PPC::LFSX:
4247 case PPC::LFDX:
4248 case PPC::STBX:
4249 case PPC::STBX8:
4250 case PPC::STHX:
4251 case PPC::STHX8:
4252 case PPC::STWX:
4253 case PPC::STWX8:
4254 case PPC::STDX:
4255 case PPC::STFSX:
4256 case PPC::STFDX:
4257 III.SignedImm = true;
4258 III.ZeroIsSpecialOrig = 1;
4259 III.ZeroIsSpecialNew = 2;
4260 III.IsCommutative = true;
4261 III.IsSummingOperands = true;
4262 III.ImmOpNo = 1;
4263 III.OpNoForForwarding = 2;
4264 switch(Opc) {
4265 default: llvm_unreachable("Unknown opcode");
4266 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4267 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4268 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4269 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4270 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4271 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4272 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4273 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4274 case PPC::LWAX:
4275 III.ImmOpcode = PPC::LWA;
4276 III.ImmMustBeMultipleOf = 4;
4277 break;
4278 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4279 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4280 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4281 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4282 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4283 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4284 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4285 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4286 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4287 case PPC::STDX:
4288 III.ImmOpcode = PPC::STD;
4289 III.ImmMustBeMultipleOf = 4;
4290 break;
4291 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4292 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4293 }
4294 break;
4295 case PPC::LBZUX:
4296 case PPC::LBZUX8:
4297 case PPC::LHZUX:
4298 case PPC::LHZUX8:
4299 case PPC::LHAUX:
4300 case PPC::LHAUX8:
4301 case PPC::LWZUX:
4302 case PPC::LWZUX8:
4303 case PPC::LDUX:
4304 case PPC::LFSUX:
4305 case PPC::LFDUX:
4306 case PPC::STBUX:
4307 case PPC::STBUX8:
4308 case PPC::STHUX:
4309 case PPC::STHUX8:
4310 case PPC::STWUX:
4311 case PPC::STWUX8:
4312 case PPC::STDUX:
4313 case PPC::STFSUX:
4314 case PPC::STFDUX:
4315 III.SignedImm = true;
4316 III.ZeroIsSpecialOrig = 2;
4317 III.ZeroIsSpecialNew = 3;
4318 III.IsCommutative = false;
4319 III.IsSummingOperands = true;
4320 III.ImmOpNo = 2;
4321 III.OpNoForForwarding = 3;
4322 switch(Opc) {
4323 default: llvm_unreachable("Unknown opcode");
4324 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4325 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4326 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4327 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4328 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4329 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4330 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4331 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4332 case PPC::LDUX:
4333 III.ImmOpcode = PPC::LDU;
4334 III.ImmMustBeMultipleOf = 4;
4335 break;
4336 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4337 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4338 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4339 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4340 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4341 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4342 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4343 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4344 case PPC::STDUX:
4345 III.ImmOpcode = PPC::STDU;
4346 III.ImmMustBeMultipleOf = 4;
4347 break;
4348 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4349 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4350 }
4351 break;
4352 // Power9 and up only. For some of these, the X-Form version has access to all
4353 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4354 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4355 // into or stored from is one of the VR registers.
4356 case PPC::LXVX:
4357 case PPC::LXSSPX:
4358 case PPC::LXSDX:
4359 case PPC::STXVX:
4360 case PPC::STXSSPX:
4361 case PPC::STXSDX:
4362 case PPC::XFLOADf32:
4363 case PPC::XFLOADf64:
4364 case PPC::XFSTOREf32:
4365 case PPC::XFSTOREf64:
4366 if (!Subtarget.hasP9Vector())
4367 return false;
4368 III.SignedImm = true;
4369 III.ZeroIsSpecialOrig = 1;
4370 III.ZeroIsSpecialNew = 2;
4371 III.IsCommutative = true;
4372 III.IsSummingOperands = true;
4373 III.ImmOpNo = 1;
4374 III.OpNoForForwarding = 2;
4375 III.ImmMustBeMultipleOf = 4;
4376 switch(Opc) {
4377 default: llvm_unreachable("Unknown opcode");
4378 case PPC::LXVX:
4379 III.ImmOpcode = PPC::LXV;
4380 III.ImmMustBeMultipleOf = 16;
4381 break;
4382 case PPC::LXSSPX:
4383 if (PostRA) {
4384 if (IsVFReg)
4385 III.ImmOpcode = PPC::LXSSP;
4386 else {
4387 III.ImmOpcode = PPC::LFS;
4388 III.ImmMustBeMultipleOf = 1;
4389 }
4390 break;
4391 }
4392 [[fallthrough]];
4393 case PPC::XFLOADf32:
4394 III.ImmOpcode = PPC::DFLOADf32;
4395 break;
4396 case PPC::LXSDX:
4397 if (PostRA) {
4398 if (IsVFReg)
4399 III.ImmOpcode = PPC::LXSD;
4400 else {
4401 III.ImmOpcode = PPC::LFD;
4402 III.ImmMustBeMultipleOf = 1;
4403 }
4404 break;
4405 }
4406 [[fallthrough]];
4407 case PPC::XFLOADf64:
4408 III.ImmOpcode = PPC::DFLOADf64;
4409 break;
4410 case PPC::STXVX:
4411 III.ImmOpcode = PPC::STXV;
4412 III.ImmMustBeMultipleOf = 16;
4413 break;
4414 case PPC::STXSSPX:
4415 if (PostRA) {
4416 if (IsVFReg)
4417 III.ImmOpcode = PPC::STXSSP;
4418 else {
4419 III.ImmOpcode = PPC::STFS;
4420 III.ImmMustBeMultipleOf = 1;
4421 }
4422 break;
4423 }
4424 [[fallthrough]];
4425 case PPC::XFSTOREf32:
4426 III.ImmOpcode = PPC::DFSTOREf32;
4427 break;
4428 case PPC::STXSDX:
4429 if (PostRA) {
4430 if (IsVFReg)
4431 III.ImmOpcode = PPC::STXSD;
4432 else {
4433 III.ImmOpcode = PPC::STFD;
4434 III.ImmMustBeMultipleOf = 1;
4435 }
4436 break;
4437 }
4438 [[fallthrough]];
4439 case PPC::XFSTOREf64:
4440 III.ImmOpcode = PPC::DFSTOREf64;
4441 break;
4442 }
4443 break;
4444 }
4445 return true;
4446}
4447
4448// Utility function for swaping two arbitrary operands of an instruction.
4449static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4450 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4451
4452 unsigned MaxOp = std::max(a: Op1, b: Op2);
4453 unsigned MinOp = std::min(a: Op1, b: Op2);
4454 MachineOperand MOp1 = MI.getOperand(i: MinOp);
4455 MachineOperand MOp2 = MI.getOperand(i: MaxOp);
4456 MI.removeOperand(OpNo: std::max(a: Op1, b: Op2));
4457 MI.removeOperand(OpNo: std::min(a: Op1, b: Op2));
4458
4459 // If the operands we are swapping are the two at the end (the common case)
4460 // we can just remove both and add them in the opposite order.
4461 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4462 MI.addOperand(Op: MOp2);
4463 MI.addOperand(Op: MOp1);
4464 } else {
4465 // Store all operands in a temporary vector, remove them and re-add in the
4466 // right order.
4467 SmallVector<MachineOperand, 2> MOps;
4468 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4469 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4470 MOps.push_back(Elt: MI.getOperand(i));
4471 MI.removeOperand(OpNo: i);
4472 }
4473 // MOp2 needs to be added next.
4474 MI.addOperand(Op: MOp2);
4475 // Now add the rest.
4476 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4477 if (i == MaxOp)
4478 MI.addOperand(Op: MOp1);
4479 else {
4480 MI.addOperand(Op: MOps.back());
4481 MOps.pop_back();
4482 }
4483 }
4484 }
4485}
4486
4487// Check if the 'MI' that has the index OpNoForForwarding
4488// meets the requirement described in the ImmInstrInfo.
4489bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4490 const ImmInstrInfo &III,
4491 unsigned OpNoForForwarding
4492 ) const {
4493 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4494 // would not work pre-RA, we can only do the check post RA.
4495 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4496 if (MRI.isSSA())
4497 return false;
4498
4499 // Cannot do the transform if MI isn't summing the operands.
4500 if (!III.IsSummingOperands)
4501 return false;
4502
4503 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4504 if (!III.ZeroIsSpecialOrig)
4505 return false;
4506
4507 // We cannot do the transform if the operand we are trying to replace
4508 // isn't the same as the operand the instruction allows.
4509 if (OpNoForForwarding != III.OpNoForForwarding)
4510 return false;
4511
4512 // Check if the instruction we are trying to transform really has
4513 // the special zero register as its operand.
4514 if (MI.getOperand(i: III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4515 MI.getOperand(i: III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4516 return false;
4517
4518 // This machine instruction is convertible if it is,
4519 // 1. summing the operands.
4520 // 2. one of the operands is special zero register.
4521 // 3. the operand we are trying to replace is allowed by the MI.
4522 return true;
4523}
4524
4525// Check if the DefMI is the add inst and set the ImmMO and RegMO
4526// accordingly.
4527bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4528 const ImmInstrInfo &III,
4529 MachineOperand *&ImmMO,
4530 MachineOperand *&RegMO) const {
4531 unsigned Opc = DefMI.getOpcode();
4532 if (Opc != PPC::ADDItocL8 && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4533 return false;
4534
4535 // Skip the optimization of transformTo[NewImm|Imm]FormFedByAdd for ADDItocL8
4536 // on AIX which is used for toc-data access. TODO: Follow up to see if it can
4537 // apply for AIX toc-data as well.
4538 if (Opc == PPC::ADDItocL8 && Subtarget.isAIX())
4539 return false;
4540
4541 assert(DefMI.getNumOperands() >= 3 &&
4542 "Add inst must have at least three operands");
4543 RegMO = &DefMI.getOperand(i: 1);
4544 ImmMO = &DefMI.getOperand(i: 2);
4545
4546 // Before RA, ADDI first operand could be a frame index.
4547 if (!RegMO->isReg())
4548 return false;
4549
4550 // This DefMI is elgible for forwarding if it is:
4551 // 1. add inst
4552 // 2. one of the operands is Imm/CPI/Global.
4553 return isAnImmediateOperand(MO: *ImmMO);
4554}
4555
4556bool PPCInstrInfo::isRegElgibleForForwarding(
4557 const MachineOperand &RegMO, const MachineInstr &DefMI,
4558 const MachineInstr &MI, bool KillDefMI,
4559 bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4560 // x = addi y, imm
4561 // ...
4562 // z = lfdx 0, x -> z = lfd imm(y)
4563 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4564 // of "y" between the DEF of "x" and "z".
4565 // The query is only valid post RA.
4566 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4567 if (MRI.isSSA())
4568 return false;
4569
4570 Register Reg = RegMO.getReg();
4571
4572 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4573 MachineBasicBlock::const_reverse_iterator It = MI;
4574 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4575 It++;
4576 for (; It != E; ++It) {
4577 if (It->modifiesRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4578 return false;
4579 else if (It->killsRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4580 IsFwdFeederRegKilled = true;
4581 if (It->readsRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4582 SeenIntermediateUse = true;
4583 // Made it to DefMI without encountering a clobber.
4584 if ((&*It) == &DefMI)
4585 break;
4586 }
4587 assert((&*It) == &DefMI && "DefMI is missing");
4588
4589 // If DefMI also defines the register to be forwarded, we can only forward it
4590 // if DefMI is being erased.
4591 if (DefMI.modifiesRegister(Reg, TRI: &getRegisterInfo()))
4592 return KillDefMI;
4593
4594 return true;
4595}
4596
4597bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4598 const MachineInstr &DefMI,
4599 const ImmInstrInfo &III,
4600 int64_t &Imm,
4601 int64_t BaseImm) const {
4602 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4603 if (DefMI.getOpcode() == PPC::ADDItocL8) {
4604 // The operand for ADDItocL8 is CPI, which isn't imm at compiling time,
4605 // However, we know that, it is 16-bit width, and has the alignment of 4.
4606 // Check if the instruction met the requirement.
4607 if (III.ImmMustBeMultipleOf > 4 ||
4608 III.TruncateImmTo || III.ImmWidth != 16)
4609 return false;
4610
4611 // Going from XForm to DForm loads means that the displacement needs to be
4612 // not just an immediate but also a multiple of 4, or 16 depending on the
4613 // load. A DForm load cannot be represented if it is a multiple of say 2.
4614 // XForm loads do not have this restriction.
4615 if (ImmMO.isGlobal()) {
4616 const DataLayout &DL = ImmMO.getGlobal()->getDataLayout();
4617 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
4618 return false;
4619 }
4620
4621 return true;
4622 }
4623
4624 if (ImmMO.isImm()) {
4625 // It is Imm, we need to check if the Imm fit the range.
4626 // Sign-extend to 64-bits.
4627 // DefMI may be folded with another imm form instruction, the result Imm is
4628 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4629 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4630 if (III.SignedImm && !ActualValue.isSignedIntN(N: III.ImmWidth))
4631 return false;
4632 if (!III.SignedImm && !ActualValue.isIntN(N: III.ImmWidth))
4633 return false;
4634 Imm = SignExtend64<16>(x: ImmMO.getImm() + BaseImm);
4635
4636 if (Imm % III.ImmMustBeMultipleOf)
4637 return false;
4638 if (III.TruncateImmTo)
4639 Imm &= ((1 << III.TruncateImmTo) - 1);
4640 }
4641 else
4642 return false;
4643
4644 // This ImmMO is forwarded if it meets the requriement describle
4645 // in ImmInstrInfo
4646 return true;
4647}
4648
4649bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4650 unsigned OpNoForForwarding,
4651 MachineInstr **KilledDef,
4652 SmallSet<Register, 4> *RegsToUpdate) const {
4653 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4654 !DefMI.getOperand(i: 1).isImm())
4655 return false;
4656
4657 MachineFunction *MF = MI.getParent()->getParent();
4658 MachineRegisterInfo *MRI = &MF->getRegInfo();
4659 bool PostRA = !MRI->isSSA();
4660
4661 int64_t Immediate = DefMI.getOperand(i: 1).getImm();
4662 // Sign-extend to 64-bits.
4663 int64_t SExtImm = SignExtend64<16>(x: Immediate);
4664
4665 bool ReplaceWithLI = false;
4666 bool Is64BitLI = false;
4667 int64_t NewImm = 0;
4668 bool SetCR = false;
4669 unsigned Opc = MI.getOpcode();
4670 switch (Opc) {
4671 default:
4672 return false;
4673
4674 // FIXME: Any branches conditional on such a comparison can be made
4675 // unconditional. At this time, this happens too infrequently to be worth
4676 // the implementation effort, but if that ever changes, we could convert
4677 // such a pattern here.
4678 case PPC::CMPWI:
4679 case PPC::CMPLWI:
4680 case PPC::CMPDI:
4681 case PPC::CMPLDI: {
4682 // Doing this post-RA would require dataflow analysis to reliably find uses
4683 // of the CR register set by the compare.
4684 // No need to fixup killed/dead flag since this transformation is only valid
4685 // before RA.
4686 if (PostRA)
4687 return false;
4688 // If a compare-immediate is fed by an immediate and is itself an input of
4689 // an ISEL (the most common case) into a COPY of the correct register.
4690 bool Changed = false;
4691 Register DefReg = MI.getOperand(i: 0).getReg();
4692 int64_t Comparand = MI.getOperand(i: 2).getImm();
4693 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4694 ? (Comparand | 0xFFFFFFFFFFFF0000)
4695 : Comparand;
4696
4697 for (auto &CompareUseMI : MRI->use_instructions(Reg: DefReg)) {
4698 unsigned UseOpc = CompareUseMI.getOpcode();
4699 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4700 continue;
4701 unsigned CRSubReg = CompareUseMI.getOperand(i: 3).getSubReg();
4702 Register TrueReg = CompareUseMI.getOperand(i: 1).getReg();
4703 Register FalseReg = CompareUseMI.getOperand(i: 2).getReg();
4704 unsigned RegToCopy =
4705 selectReg(Imm1: SExtImm, Imm2: SExtComparand, CompareOpc: Opc, TrueReg, FalseReg, CRSubReg);
4706 if (RegToCopy == PPC::NoRegister)
4707 continue;
4708 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4709 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4710 CompareUseMI.setDesc(get(Opcode: UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4711 replaceInstrOperandWithImm(MI&: CompareUseMI, OpNo: 1, Imm: 0);
4712 CompareUseMI.removeOperand(OpNo: 3);
4713 CompareUseMI.removeOperand(OpNo: 2);
4714 continue;
4715 }
4716 LLVM_DEBUG(
4717 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4718 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4719 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4720 if (RegsToUpdate) {
4721 for (const MachineOperand &MO : CompareUseMI.operands())
4722 if (MO.isReg())
4723 RegsToUpdate->insert(V: MO.getReg());
4724 }
4725 // Convert to copy and remove unneeded operands.
4726 CompareUseMI.setDesc(get(Opcode: PPC::COPY));
4727 CompareUseMI.removeOperand(OpNo: 3);
4728 CompareUseMI.removeOperand(OpNo: RegToCopy == TrueReg ? 2 : 1);
4729 CmpIselsConverted++;
4730 Changed = true;
4731 LLVM_DEBUG(CompareUseMI.dump());
4732 }
4733 if (Changed)
4734 return true;
4735 // This may end up incremented multiple times since this function is called
4736 // during a fixed-point transformation, but it is only meant to indicate the
4737 // presence of this opportunity.
4738 MissedConvertibleImmediateInstrs++;
4739 return false;
4740 }
4741
4742 // Immediate forms - may simply be convertable to an LI.
4743 case PPC::ADDI:
4744 case PPC::ADDI8: {
4745 // Does the sum fit in a 16-bit signed field?
4746 int64_t Addend = MI.getOperand(i: 2).getImm();
4747 if (isInt<16>(x: Addend + SExtImm)) {
4748 ReplaceWithLI = true;
4749 Is64BitLI = Opc == PPC::ADDI8;
4750 NewImm = Addend + SExtImm;
4751 break;
4752 }
4753 return false;
4754 }
4755 case PPC::SUBFIC:
4756 case PPC::SUBFIC8: {
4757 // Only transform this if the CARRY implicit operand is dead.
4758 if (MI.getNumOperands() > 3 && !MI.getOperand(i: 3).isDead())
4759 return false;
4760 int64_t Minuend = MI.getOperand(i: 2).getImm();
4761 if (isInt<16>(x: Minuend - SExtImm)) {
4762 ReplaceWithLI = true;
4763 Is64BitLI = Opc == PPC::SUBFIC8;
4764 NewImm = Minuend - SExtImm;
4765 break;
4766 }
4767 return false;
4768 }
4769 case PPC::RLDICL:
4770 case PPC::RLDICL_rec:
4771 case PPC::RLDICL_32:
4772 case PPC::RLDICL_32_64: {
4773 // Use APInt's rotate function.
4774 int64_t SH = MI.getOperand(i: 2).getImm();
4775 int64_t MB = MI.getOperand(i: 3).getImm();
4776 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4777 SExtImm, true);
4778 InVal = InVal.rotl(rotateAmt: SH);
4779 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4780 InVal &= Mask;
4781 // Can't replace negative values with an LI as that will sign-extend
4782 // and not clear the left bits. If we're setting the CR bit, we will use
4783 // ANDI_rec which won't sign extend, so that's safe.
4784 if (isUInt<15>(x: InVal.getSExtValue()) ||
4785 (Opc == PPC::RLDICL_rec && isUInt<16>(x: InVal.getSExtValue()))) {
4786 ReplaceWithLI = true;
4787 Is64BitLI = Opc != PPC::RLDICL_32;
4788 NewImm = InVal.getSExtValue();
4789 SetCR = Opc == PPC::RLDICL_rec;
4790 break;
4791 }
4792 return false;
4793 }
4794 case PPC::RLWINM:
4795 case PPC::RLWINM8:
4796 case PPC::RLWINM_rec:
4797 case PPC::RLWINM8_rec: {
4798 int64_t SH = MI.getOperand(i: 2).getImm();
4799 int64_t MB = MI.getOperand(i: 3).getImm();
4800 int64_t ME = MI.getOperand(i: 4).getImm();
4801 APInt InVal(32, SExtImm, true);
4802 InVal = InVal.rotl(rotateAmt: SH);
4803 APInt Mask = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - ME - 1, hiBit: 32 - MB);
4804 InVal &= Mask;
4805 // Can't replace negative values with an LI as that will sign-extend
4806 // and not clear the left bits. If we're setting the CR bit, we will use
4807 // ANDI_rec which won't sign extend, so that's safe.
4808 bool ValueFits = isUInt<15>(x: InVal.getSExtValue());
4809 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4810 isUInt<16>(x: InVal.getSExtValue()));
4811 if (ValueFits) {
4812 ReplaceWithLI = true;
4813 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4814 NewImm = InVal.getSExtValue();
4815 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4816 break;
4817 }
4818 return false;
4819 }
4820 case PPC::ORI:
4821 case PPC::ORI8:
4822 case PPC::XORI:
4823 case PPC::XORI8: {
4824 int64_t LogicalImm = MI.getOperand(i: 2).getImm();
4825 int64_t Result = 0;
4826 if (Opc == PPC::ORI || Opc == PPC::ORI8)
4827 Result = LogicalImm | SExtImm;
4828 else
4829 Result = LogicalImm ^ SExtImm;
4830 if (isInt<16>(x: Result)) {
4831 ReplaceWithLI = true;
4832 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4833 NewImm = Result;
4834 break;
4835 }
4836 return false;
4837 }
4838 }
4839
4840 if (ReplaceWithLI) {
4841 // We need to be careful with CR-setting instructions we're replacing.
4842 if (SetCR) {
4843 // We don't know anything about uses when we're out of SSA, so only
4844 // replace if the new immediate will be reproduced.
4845 bool ImmChanged = (SExtImm & NewImm) != NewImm;
4846 if (PostRA && ImmChanged)
4847 return false;
4848
4849 if (!PostRA) {
4850 // If the defining load-immediate has no other uses, we can just replace
4851 // the immediate with the new immediate.
4852 if (MRI->hasOneUse(RegNo: DefMI.getOperand(i: 0).getReg()))
4853 DefMI.getOperand(i: 1).setImm(NewImm);
4854
4855 // If we're not using the GPR result of the CR-setting instruction, we
4856 // just need to and with zero/non-zero depending on the new immediate.
4857 else if (MRI->use_empty(RegNo: MI.getOperand(i: 0).getReg())) {
4858 if (NewImm) {
4859 assert(Immediate && "Transformation converted zero to non-zero?");
4860 NewImm = Immediate;
4861 }
4862 } else if (ImmChanged)
4863 return false;
4864 }
4865 }
4866
4867 LLVM_DEBUG(dbgs() << "Replacing constant instruction:\n");
4868 LLVM_DEBUG(MI.dump());
4869 LLVM_DEBUG(dbgs() << "Fed by:\n");
4870 LLVM_DEBUG(DefMI.dump());
4871 LoadImmediateInfo LII;
4872 LII.Imm = NewImm;
4873 LII.Is64Bit = Is64BitLI;
4874 LII.SetCR = SetCR;
4875 // If we're setting the CR, the original load-immediate must be kept (as an
4876 // operand to ANDI_rec/ANDI8_rec).
4877 if (KilledDef && SetCR)
4878 *KilledDef = nullptr;
4879 replaceInstrWithLI(MI, LII);
4880
4881 if (PostRA)
4882 recomputeLivenessFlags(MBB&: *MI.getParent());
4883
4884 LLVM_DEBUG(dbgs() << "With:\n");
4885 LLVM_DEBUG(MI.dump());
4886 return true;
4887 }
4888 return false;
4889}
4890
4891bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4892 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4893 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4894 bool PostRA = !MRI->isSSA();
4895 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4896 // for post-ra.
4897 if (PostRA)
4898 return false;
4899
4900 // Only handle load/store.
4901 if (!MI.mayLoadOrStore())
4902 return false;
4903
4904 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(ImmOpcode: MI.getOpcode());
4905
4906 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4907 "MI must have x-form opcode");
4908
4909 // get Imm Form info.
4910 ImmInstrInfo III;
4911 bool IsVFReg = MI.getOperand(i: 0).isReg() &&
4912 MI.getOperand(i: 0).getReg().isPhysical() &&
4913 PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg());
4914
4915 if (!instrHasImmForm(Opc: XFormOpcode, IsVFReg, III, PostRA))
4916 return false;
4917
4918 if (!III.IsSummingOperands)
4919 return false;
4920
4921 if (OpNoForForwarding != III.OpNoForForwarding)
4922 return false;
4923
4924 MachineOperand ImmOperandMI = MI.getOperand(i: III.ImmOpNo);
4925 if (!ImmOperandMI.isImm())
4926 return false;
4927
4928 // Check DefMI.
4929 MachineOperand *ImmMO = nullptr;
4930 MachineOperand *RegMO = nullptr;
4931 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4932 return false;
4933 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4934
4935 // Check Imm.
4936 // Set ImmBase from imm instruction as base and get new Imm inside
4937 // isImmElgibleForForwarding.
4938 int64_t ImmBase = ImmOperandMI.getImm();
4939 int64_t Imm = 0;
4940 if (!isImmElgibleForForwarding(ImmMO: *ImmMO, DefMI, III, Imm, BaseImm: ImmBase))
4941 return false;
4942
4943 // Do the transform
4944 LLVM_DEBUG(dbgs() << "Replacing existing reg+imm instruction:\n");
4945 LLVM_DEBUG(MI.dump());
4946 LLVM_DEBUG(dbgs() << "Fed by:\n");
4947 LLVM_DEBUG(DefMI.dump());
4948
4949 MI.getOperand(i: III.OpNoForForwarding).setReg(RegMO->getReg());
4950 MI.getOperand(i: III.ImmOpNo).setImm(Imm);
4951
4952 LLVM_DEBUG(dbgs() << "With:\n");
4953 LLVM_DEBUG(MI.dump());
4954 return true;
4955}
4956
4957// If an X-Form instruction is fed by an add-immediate and one of its operands
4958// is the literal zero, attempt to forward the source of the add-immediate to
4959// the corresponding D-Form instruction with the displacement coming from
4960// the immediate being added.
4961bool PPCInstrInfo::transformToImmFormFedByAdd(
4962 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4963 MachineInstr &DefMI, bool KillDefMI) const {
4964 // RegMO ImmMO
4965 // | |
4966 // x = addi reg, imm <----- DefMI
4967 // y = op 0 , x <----- MI
4968 // |
4969 // OpNoForForwarding
4970 // Check if the MI meet the requirement described in the III.
4971 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
4972 return false;
4973
4974 // Check if the DefMI meet the requirement
4975 // described in the III. If yes, set the ImmMO and RegMO accordingly.
4976 MachineOperand *ImmMO = nullptr;
4977 MachineOperand *RegMO = nullptr;
4978 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4979 return false;
4980 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4981
4982 // As we get the Imm operand now, we need to check if the ImmMO meet
4983 // the requirement described in the III. If yes set the Imm.
4984 int64_t Imm = 0;
4985 if (!isImmElgibleForForwarding(ImmMO: *ImmMO, DefMI, III, Imm))
4986 return false;
4987
4988 bool IsFwdFeederRegKilled = false;
4989 bool SeenIntermediateUse = false;
4990 // Check if the RegMO can be forwarded to MI.
4991 if (!isRegElgibleForForwarding(RegMO: *RegMO, DefMI, MI, KillDefMI,
4992 IsFwdFeederRegKilled, SeenIntermediateUse))
4993 return false;
4994
4995 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4996 bool PostRA = !MRI.isSSA();
4997
4998 // We know that, the MI and DefMI both meet the pattern, and
4999 // the Imm also meet the requirement with the new Imm-form.
5000 // It is safe to do the transformation now.
5001 LLVM_DEBUG(dbgs() << "Replacing indexed instruction:\n");
5002 LLVM_DEBUG(MI.dump());
5003 LLVM_DEBUG(dbgs() << "Fed by:\n");
5004 LLVM_DEBUG(DefMI.dump());
5005
5006 // Update the base reg first.
5007 MI.getOperand(i: III.OpNoForForwarding).ChangeToRegister(Reg: RegMO->getReg(),
5008 isDef: false, isImp: false,
5009 isKill: RegMO->isKill());
5010
5011 // Then, update the imm.
5012 if (ImmMO->isImm()) {
5013 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
5014 // directly.
5015 replaceInstrOperandWithImm(MI, OpNo: III.ZeroIsSpecialOrig, Imm);
5016 }
5017 else {
5018 // Otherwise, it is Constant Pool Index(CPI) or Global,
5019 // which is relocation in fact. We need to replace the special zero
5020 // register with ImmMO.
5021 // Before that, we need to fixup the target flags for imm.
5022 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
5023 if (DefMI.getOpcode() == PPC::ADDItocL8)
5024 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
5025
5026 // MI didn't have the interface such as MI.setOperand(i) though
5027 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
5028 // ImmMO, we need to remove ZERO operand and all the operands behind it,
5029 // and, add the ImmMO, then, move back all the operands behind ZERO.
5030 SmallVector<MachineOperand, 2> MOps;
5031 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
5032 MOps.push_back(Elt: MI.getOperand(i));
5033 MI.removeOperand(OpNo: i);
5034 }
5035
5036 // Remove the last MO in the list, which is ZERO operand in fact.
5037 MOps.pop_back();
5038 // Add the imm operand.
5039 MI.addOperand(Op: *ImmMO);
5040 // Now add the rest back.
5041 for (auto &MO : MOps)
5042 MI.addOperand(Op: MO);
5043 }
5044
5045 // Update the opcode.
5046 MI.setDesc(get(Opcode: III.ImmOpcode));
5047
5048 if (PostRA)
5049 recomputeLivenessFlags(MBB&: *MI.getParent());
5050 LLVM_DEBUG(dbgs() << "With:\n");
5051 LLVM_DEBUG(MI.dump());
5052
5053 return true;
5054}
5055
5056bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
5057 const ImmInstrInfo &III,
5058 unsigned ConstantOpNo,
5059 MachineInstr &DefMI) const {
5060 // DefMI must be LI or LI8.
5061 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
5062 !DefMI.getOperand(i: 1).isImm())
5063 return false;
5064
5065 // Get Imm operand and Sign-extend to 64-bits.
5066 int64_t Imm = SignExtend64<16>(x: DefMI.getOperand(i: 1).getImm());
5067
5068 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5069 bool PostRA = !MRI.isSSA();
5070 // Exit early if we can't convert this.
5071 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
5072 return false;
5073 if (Imm % III.ImmMustBeMultipleOf)
5074 return false;
5075 if (III.TruncateImmTo)
5076 Imm &= ((1 << III.TruncateImmTo) - 1);
5077 if (III.SignedImm) {
5078 APInt ActualValue(64, Imm, true);
5079 if (!ActualValue.isSignedIntN(N: III.ImmWidth))
5080 return false;
5081 } else {
5082 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
5083 if ((uint64_t)Imm > UnsignedMax)
5084 return false;
5085 }
5086
5087 // If we're post-RA, the instructions don't agree on whether register zero is
5088 // special, we can transform this as long as the register operand that will
5089 // end up in the location where zero is special isn't R0.
5090 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5091 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
5092 III.ZeroIsSpecialNew + 1;
5093 Register OrigZeroReg = MI.getOperand(i: PosForOrigZero).getReg();
5094 Register NewZeroReg = MI.getOperand(i: III.ZeroIsSpecialNew).getReg();
5095 // If R0 is in the operand where zero is special for the new instruction,
5096 // it is unsafe to transform if the constant operand isn't that operand.
5097 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
5098 ConstantOpNo != III.ZeroIsSpecialNew)
5099 return false;
5100 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
5101 ConstantOpNo != PosForOrigZero)
5102 return false;
5103 }
5104
5105 unsigned Opc = MI.getOpcode();
5106 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
5107 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
5108 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
5109 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
5110 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
5111 Opc == PPC::SRD || Opc == PPC::SRD_rec;
5112 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
5113 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
5114 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
5115 Opc == PPC::SRD_rec;
5116
5117 LLVM_DEBUG(dbgs() << "Replacing reg+reg instruction: ");
5118 LLVM_DEBUG(MI.dump());
5119 LLVM_DEBUG(dbgs() << "Fed by load-immediate: ");
5120 LLVM_DEBUG(DefMI.dump());
5121 MI.setDesc(get(Opcode: III.ImmOpcode));
5122 if (ConstantOpNo == III.OpNoForForwarding) {
5123 // Converting shifts to immediate form is a bit tricky since they may do
5124 // one of three things:
5125 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
5126 // 2. If the shift amount is zero, the result is unchanged (save for maybe
5127 // setting CR0)
5128 // 3. If the shift amount is in [1, OpSize), it's just a shift
5129 if (SpecialShift32 || SpecialShift64) {
5130 LoadImmediateInfo LII;
5131 LII.Imm = 0;
5132 LII.SetCR = SetCR;
5133 LII.Is64Bit = SpecialShift64;
5134 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
5135 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
5136 replaceInstrWithLI(MI, LII);
5137 // Shifts by zero don't change the value. If we don't need to set CR0,
5138 // just convert this to a COPY. Can't do this post-RA since we've already
5139 // cleaned up the copies.
5140 else if (!SetCR && ShAmt == 0 && !PostRA) {
5141 MI.removeOperand(OpNo: 2);
5142 MI.setDesc(get(Opcode: PPC::COPY));
5143 } else {
5144 // The 32 bit and 64 bit instructions are quite different.
5145 if (SpecialShift32) {
5146 // Left shifts use (N, 0, 31-N).
5147 // Right shifts use (32-N, N, 31) if 0 < N < 32.
5148 // use (0, 0, 31) if N == 0.
5149 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
5150 uint64_t MB = RightShift ? ShAmt : 0;
5151 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
5152 replaceInstrOperandWithImm(MI, OpNo: III.OpNoForForwarding, Imm: SH);
5153 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(Val: MB)
5154 .addImm(Val: ME);
5155 } else {
5156 // Left shifts use (N, 63-N).
5157 // Right shifts use (64-N, N) if 0 < N < 64.
5158 // use (0, 0) if N == 0.
5159 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
5160 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
5161 replaceInstrOperandWithImm(MI, OpNo: III.OpNoForForwarding, Imm: SH);
5162 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(Val: ME);
5163 }
5164 }
5165 } else
5166 replaceInstrOperandWithImm(MI, OpNo: ConstantOpNo, Imm);
5167 }
5168 // Convert commutative instructions (switch the operands and convert the
5169 // desired one to an immediate.
5170 else if (III.IsCommutative) {
5171 replaceInstrOperandWithImm(MI, OpNo: ConstantOpNo, Imm);
5172 swapMIOperands(MI, Op1: ConstantOpNo, Op2: III.OpNoForForwarding);
5173 } else
5174 llvm_unreachable("Should have exited early!");
5175
5176 // For instructions for which the constant register replaces a different
5177 // operand than where the immediate goes, we need to swap them.
5178 if (III.OpNoForForwarding != III.ImmOpNo)
5179 swapMIOperands(MI, Op1: III.OpNoForForwarding, Op2: III.ImmOpNo);
5180
5181 // If the special R0/X0 register index are different for original instruction
5182 // and new instruction, we need to fix up the register class in new
5183 // instruction.
5184 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5185 if (III.ZeroIsSpecialNew) {
5186 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
5187 // need to fix up register class.
5188 Register RegToModify = MI.getOperand(i: III.ZeroIsSpecialNew).getReg();
5189 if (RegToModify.isVirtual()) {
5190 const TargetRegisterClass *NewRC =
5191 MRI.getRegClass(Reg: RegToModify)->hasSuperClassEq(RC: &PPC::GPRCRegClass) ?
5192 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
5193 MRI.setRegClass(Reg: RegToModify, RC: NewRC);
5194 }
5195 }
5196 }
5197
5198 if (PostRA)
5199 recomputeLivenessFlags(MBB&: *MI.getParent());
5200
5201 LLVM_DEBUG(dbgs() << "With: ");
5202 LLVM_DEBUG(MI.dump());
5203 LLVM_DEBUG(dbgs() << "\n");
5204 return true;
5205}
5206
5207const TargetRegisterClass *
5208PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
5209 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
5210 return &PPC::VSRCRegClass;
5211 return RC;
5212}
5213
5214int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
5215 return PPC::getRecordFormOpcode(Opcode);
5216}
5217
5218static bool isOpZeroOfSubwordPreincLoad(int Opcode) {
5219 return (Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 ||
5220 Opcode == PPC::LBZUX8 || Opcode == PPC::LHZU ||
5221 Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 ||
5222 Opcode == PPC::LHZUX8);
5223}
5224
5225// This function checks for sign extension from 32 bits to 64 bits.
5226static bool definedBySignExtendingOp(const PPCInstrInfo &TII,
5227 const unsigned Reg,
5228 const MachineRegisterInfo *MRI) {
5229 if (!Register::isVirtualRegister(Reg))
5230 return false;
5231
5232 MachineInstr *MI = MRI->getVRegDef(Reg);
5233 if (!MI)
5234 return false;
5235
5236 int Opcode = MI->getOpcode();
5237 if (TII.isSExt32To64(Opcode))
5238 return true;
5239
5240 // The first def of LBZU/LHZU is sign extended.
5241 if (isOpZeroOfSubwordPreincLoad(Opcode) && MI->getOperand(i: 0).getReg() == Reg)
5242 return true;
5243
5244 // RLDICL generates sign-extended output if it clears at least
5245 // 33 bits from the left (MSB).
5246 if (Opcode == PPC::RLDICL && MI->getOperand(i: 3).getImm() >= 33)
5247 return true;
5248
5249 // If at least one bit from left in a lower word is masked out,
5250 // all of 0 to 32-th bits of the output are cleared.
5251 // Hence the output is already sign extended.
5252 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5253 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
5254 MI->getOperand(i: 3).getImm() > 0 &&
5255 MI->getOperand(i: 3).getImm() <= MI->getOperand(i: 4).getImm())
5256 return true;
5257
5258 // If the most significant bit of immediate in ANDIS is zero,
5259 // all of 0 to 32-th bits are cleared.
5260 if (Opcode == PPC::ANDIS_rec || Opcode == PPC::ANDIS8_rec) {
5261 uint16_t Imm = MI->getOperand(i: 2).getImm();
5262 if ((Imm & 0x8000) == 0)
5263 return true;
5264 }
5265
5266 return false;
5267}
5268
5269// This function checks the machine instruction that defines the input register
5270// Reg. If that machine instruction always outputs a value that has only zeros
5271// in the higher 32 bits then this function will return true.
5272static bool definedByZeroExtendingOp(const PPCInstrInfo &TII,
5273 const unsigned Reg,
5274 const MachineRegisterInfo *MRI) {
5275 if (!Register::isVirtualRegister(Reg))
5276 return false;
5277
5278 MachineInstr *MI = MRI->getVRegDef(Reg);
5279 if (!MI)
5280 return false;
5281
5282 int Opcode = MI->getOpcode();
5283 if (TII.isZExt32To64(Opcode))
5284 return true;
5285
5286 // The first def of LBZU/LHZU/LWZU are zero extended.
5287 if ((isOpZeroOfSubwordPreincLoad(Opcode) || Opcode == PPC::LWZU ||
5288 Opcode == PPC::LWZUX || Opcode == PPC::LWZU8 || Opcode == PPC::LWZUX8) &&
5289 MI->getOperand(i: 0).getReg() == Reg)
5290 return true;
5291
5292 // The 16-bit immediate is sign-extended in li/lis.
5293 // If the most significant bit is zero, all higher bits are zero.
5294 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
5295 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
5296 int64_t Imm = MI->getOperand(i: 1).getImm();
5297 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
5298 return true;
5299 }
5300
5301 // We have some variations of rotate-and-mask instructions
5302 // that clear higher 32-bits.
5303 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
5304 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
5305 Opcode == PPC::RLDICL_32_64) &&
5306 MI->getOperand(i: 3).getImm() >= 32)
5307 return true;
5308
5309 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
5310 MI->getOperand(i: 3).getImm() >= 32 &&
5311 MI->getOperand(i: 3).getImm() <= 63 - MI->getOperand(i: 2).getImm())
5312 return true;
5313
5314 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5315 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
5316 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
5317 MI->getOperand(i: 3).getImm() <= MI->getOperand(i: 4).getImm())
5318 return true;
5319
5320 return false;
5321}
5322
5323// This function returns true if the input MachineInstr is a TOC save
5324// instruction.
5325bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
5326 if (!MI.getOperand(i: 1).isImm() || !MI.getOperand(i: 2).isReg())
5327 return false;
5328 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5329 unsigned StackOffset = MI.getOperand(i: 1).getImm();
5330 Register StackReg = MI.getOperand(i: 2).getReg();
5331 Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
5332 if (StackReg == SPReg && StackOffset == TOCSaveOffset)
5333 return true;
5334
5335 return false;
5336}
5337
5338// We limit the max depth to track incoming values of PHIs or binary ops
5339// (e.g. AND) to avoid excessive cost.
5340const unsigned MAX_BINOP_DEPTH = 1;
5341
5342// This function will promote the instruction which defines the register `Reg`
5343// in the parameter from a 32-bit to a 64-bit instruction if needed. The logic
5344// used to check whether an instruction needs to be promoted or not is similar
5345// to the logic used to check whether or not a defined register is sign or zero
5346// extended within the function PPCInstrInfo::isSignOrZeroExtended.
5347// Additionally, the `promoteInstr32To64ForElimEXTSW` function is recursive.
5348// BinOpDepth does not count all of the recursions. The parameter BinOpDepth is
5349// incremented only when `promoteInstr32To64ForElimEXTSW` calls itself more
5350// than once. This is done to prevent exponential recursion.
5351void PPCInstrInfo::promoteInstr32To64ForElimEXTSW(const Register &Reg,
5352 MachineRegisterInfo *MRI,
5353 unsigned BinOpDepth,
5354 LiveVariables *LV) const {
5355 if (!Reg.isVirtual())
5356 return;
5357
5358 MachineInstr *MI = MRI->getVRegDef(Reg);
5359 if (!MI)
5360 return;
5361
5362 unsigned Opcode = MI->getOpcode();
5363
5364 switch (Opcode) {
5365 case PPC::OR:
5366 case PPC::ISEL:
5367 case PPC::OR8:
5368 case PPC::PHI: {
5369 if (BinOpDepth >= MAX_BINOP_DEPTH)
5370 break;
5371 unsigned OperandEnd = 3, OperandStride = 1;
5372 if (Opcode == PPC::PHI) {
5373 OperandEnd = MI->getNumOperands();
5374 OperandStride = 2;
5375 }
5376
5377 for (unsigned I = 1; I < OperandEnd; I += OperandStride) {
5378 assert(MI->getOperand(I).isReg() && "Operand must be register");
5379 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: I).getReg(), MRI,
5380 BinOpDepth: BinOpDepth + 1, LV);
5381 }
5382
5383 break;
5384 }
5385 case PPC::COPY: {
5386 // Refers to the logic of the `case PPC::COPY` statement in the function
5387 // PPCInstrInfo::isSignOrZeroExtended().
5388
5389 Register SrcReg = MI->getOperand(i: 1).getReg();
5390 // In both ELFv1 and v2 ABI, method parameters and the return value
5391 // are sign- or zero-extended.
5392 const MachineFunction *MF = MI->getMF();
5393 if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5394 // If this is a copy from another register, we recursively promote the
5395 // source.
5396 promoteInstr32To64ForElimEXTSW(Reg: SrcReg, MRI, BinOpDepth, LV);
5397 return;
5398 }
5399
5400 // From here on everything is SVR4ABI. COPY will be eliminated in the other
5401 // pass, we do not need promote the COPY pseudo opcode.
5402
5403 if (SrcReg != PPC::X3)
5404 // If this is a copy from another register, we recursively promote the
5405 // source.
5406 promoteInstr32To64ForElimEXTSW(Reg: SrcReg, MRI, BinOpDepth, LV);
5407 return;
5408 }
5409 case PPC::ORI:
5410 case PPC::XORI:
5411 case PPC::ORIS:
5412 case PPC::XORIS:
5413 case PPC::ORI8:
5414 case PPC::XORI8:
5415 case PPC::ORIS8:
5416 case PPC::XORIS8:
5417 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 1).getReg(), MRI, BinOpDepth,
5418 LV);
5419 break;
5420 case PPC::AND:
5421 case PPC::AND8:
5422 if (BinOpDepth >= MAX_BINOP_DEPTH)
5423 break;
5424
5425 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 1).getReg(), MRI,
5426 BinOpDepth: BinOpDepth + 1, LV);
5427 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 2).getReg(), MRI,
5428 BinOpDepth: BinOpDepth + 1, LV);
5429 break;
5430 }
5431
5432 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
5433 if (RC == &PPC::G8RCRegClass || RC == &PPC::G8RC_and_G8RC_NOX0RegClass)
5434 return;
5435
5436 // Map the 32bit to 64bit opcodes for instructions that are not signed or zero
5437 // extended themselves, but may have operands who's destination registers of
5438 // signed or zero extended instructions.
5439 DenseMap<unsigned, unsigned> OpcodeMap = {
5440 {PPC::OR, PPC::OR8}, {PPC::ISEL, PPC::ISEL8},
5441 {PPC::ORI, PPC::ORI8}, {PPC::XORI, PPC::XORI8},
5442 {PPC::ORIS, PPC::ORIS8}, {PPC::XORIS, PPC::XORIS8},
5443 {PPC::AND, PPC::AND8}};
5444
5445 int NewOpcode = -1;
5446 auto It = OpcodeMap.find(Val: Opcode);
5447 if (It != OpcodeMap.end()) {
5448 // Set the new opcode to the mapped 64-bit version.
5449 NewOpcode = It->second;
5450 } else {
5451 if (!isSExt32To64(Opcode))
5452 return;
5453
5454 // The TableGen function `get64BitInstrFromSignedExt32BitInstr` is used to
5455 // map the 32-bit instruction with the `SExt32To64` flag to the 64-bit
5456 // instruction with the same opcode.
5457 NewOpcode = PPC::get64BitInstrFromSignedExt32BitInstr(Opcode);
5458 }
5459
5460 assert(NewOpcode != -1 &&
5461 "Must have a 64-bit opcode to map the 32-bit opcode!");
5462
5463 const MCInstrDesc &MCID = get(Opcode: NewOpcode);
5464 const TargetRegisterClass *NewRC =
5465 RI.getRegClass(i: MCID.operands()[0].RegClass);
5466
5467 Register SrcReg = MI->getOperand(i: 0).getReg();
5468 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
5469
5470 // If the register class of the defined register in the 32-bit instruction
5471 // is the same as the register class of the defined register in the promoted
5472 // 64-bit instruction, we do not need to promote the instruction.
5473 if (NewRC == SrcRC)
5474 return;
5475
5476 DebugLoc DL = MI->getDebugLoc();
5477 auto MBB = MI->getParent();
5478
5479 // Since the pseudo-opcode of the instruction is promoted from 32-bit to
5480 // 64-bit, if the source reg class of the original instruction belongs to
5481 // PPC::GRCRegClass or PPC::GPRC_and_GPRC_NOR0RegClass, we need to promote
5482 // the operand to PPC::G8CRegClass or PPC::G8RC_and_G8RC_NOR0RegClass,
5483 // respectively.
5484 SmallVector<Register> PromoteRegs(MI->getNumOperands());
5485 for (unsigned i = 1; i < MI->getNumOperands(); i++) {
5486 MachineOperand &Operand = MI->getOperand(i);
5487 if (!Operand.isReg())
5488 continue;
5489
5490 Register OperandReg = Operand.getReg();
5491 if (!OperandReg.isVirtual())
5492 continue;
5493
5494 const TargetRegisterClass *NewUsedRegRC =
5495 RI.getRegClass(i: MCID.operands()[i].RegClass);
5496 const TargetRegisterClass *OrgRC = MRI->getRegClass(Reg: OperandReg);
5497 if (NewUsedRegRC != OrgRC && (OrgRC == &PPC::GPRCRegClass ||
5498 OrgRC == &PPC::GPRC_and_GPRC_NOR0RegClass)) {
5499 // Promote the used 32-bit register to 64-bit register.
5500 Register TmpReg = MRI->createVirtualRegister(RegClass: NewUsedRegRC);
5501 Register DstTmpReg = MRI->createVirtualRegister(RegClass: NewUsedRegRC);
5502 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: PPC::IMPLICIT_DEF), DestReg: TmpReg);
5503 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: PPC::INSERT_SUBREG), DestReg: DstTmpReg)
5504 .addReg(RegNo: TmpReg)
5505 .addReg(RegNo: OperandReg)
5506 .addImm(Val: PPC::sub_32);
5507 PromoteRegs[i] = DstTmpReg;
5508 }
5509 }
5510
5511 Register NewDefinedReg = MRI->createVirtualRegister(RegClass: NewRC);
5512
5513 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDefinedReg);
5514 MachineBasicBlock::instr_iterator Iter(MI);
5515 --Iter;
5516 MachineInstrBuilder MIBuilder(*Iter->getMF(), Iter);
5517 for (unsigned i = 1; i < MI->getNumOperands(); i++) {
5518 if (PromoteRegs[i])
5519 MIBuilder.addReg(RegNo: PromoteRegs[i], Flags: RegState::Kill);
5520 else
5521 Iter->addOperand(Op: MI->getOperand(i));
5522 }
5523
5524 for (unsigned i = 1; i < Iter->getNumOperands(); i++) {
5525 MachineOperand &Operand = Iter->getOperand(i);
5526 if (!Operand.isReg())
5527 continue;
5528 Register OperandReg = Operand.getReg();
5529 if (!OperandReg.isVirtual())
5530 continue;
5531 LV->recomputeForSingleDefVirtReg(Reg: OperandReg);
5532 }
5533
5534 MI->eraseFromParent();
5535
5536 // A defined register may be used by other instructions that are 32-bit.
5537 // After the defined register is promoted to 64-bit for the promoted
5538 // instruction, we need to demote the 64-bit defined register back to a
5539 // 32-bit register
5540 BuildMI(BB&: *MBB, I: ++Iter, MIMD: DL, MCID: get(Opcode: PPC::COPY), DestReg: SrcReg)
5541 .addReg(RegNo: NewDefinedReg, Flags: RegState::Kill, SubReg: PPC::sub_32);
5542 LV->recomputeForSingleDefVirtReg(Reg: NewDefinedReg);
5543}
5544
5545// The isSignOrZeroExtended function is recursive. The parameter BinOpDepth
5546// does not count all of the recursions. The parameter BinOpDepth is incremented
5547// only when isSignOrZeroExtended calls itself more than once. This is done to
5548// prevent expontential recursion. There is no parameter to track linear
5549// recursion.
5550std::pair<bool, bool>
5551PPCInstrInfo::isSignOrZeroExtended(const unsigned Reg,
5552 const unsigned BinOpDepth,
5553 const MachineRegisterInfo *MRI) const {
5554 if (!Register::isVirtualRegister(Reg))
5555 return std::pair<bool, bool>(false, false);
5556
5557 MachineInstr *MI = MRI->getVRegDef(Reg);
5558 if (!MI)
5559 return std::pair<bool, bool>(false, false);
5560
5561 bool IsSExt = definedBySignExtendingOp(TII: *this, Reg, MRI);
5562 bool IsZExt = definedByZeroExtendingOp(TII: *this, Reg, MRI);
5563
5564 // If we know the instruction always returns sign- and zero-extended result,
5565 // return here.
5566 if (IsSExt && IsZExt)
5567 return std::pair<bool, bool>(IsSExt, IsZExt);
5568
5569 switch (MI->getOpcode()) {
5570 case PPC::COPY: {
5571 Register SrcReg = MI->getOperand(i: 1).getReg();
5572
5573 // In both ELFv1 and v2 ABI, method parameters and the return value
5574 // are sign- or zero-extended.
5575 const MachineFunction *MF = MI->getMF();
5576
5577 if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5578 // If this is a copy from another register, we recursively check source.
5579 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5580 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5581 SrcExt.second || IsZExt);
5582 }
5583
5584 // From here on everything is SVR4ABI
5585 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
5586 // We check the ZExt/SExt flags for a method parameter.
5587 if (MI->getParent()->getBasicBlock() ==
5588 &MF->getFunction().getEntryBlock()) {
5589 Register VReg = MI->getOperand(i: 0).getReg();
5590 if (MF->getRegInfo().isLiveIn(Reg: VReg)) {
5591 IsSExt |= FuncInfo->isLiveInSExt(VReg);
5592 IsZExt |= FuncInfo->isLiveInZExt(VReg);
5593 return std::pair<bool, bool>(IsSExt, IsZExt);
5594 }
5595 }
5596
5597 if (SrcReg != PPC::X3) {
5598 // If this is a copy from another register, we recursively check source.
5599 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5600 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5601 SrcExt.second || IsZExt);
5602 }
5603
5604 // For a method return value, we check the ZExt/SExt flags in attribute.
5605 // We assume the following code sequence for method call.
5606 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
5607 // BL8_NOP @func,...
5608 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
5609 // %5 = COPY %x3; G8RC:%5
5610 const MachineBasicBlock *MBB = MI->getParent();
5611 std::pair<bool, bool> IsExtendPair = std::pair<bool, bool>(IsSExt, IsZExt);
5612 MachineBasicBlock::const_instr_iterator II =
5613 MachineBasicBlock::const_instr_iterator(MI);
5614 if (II == MBB->instr_begin() || (--II)->getOpcode() != PPC::ADJCALLSTACKUP)
5615 return IsExtendPair;
5616
5617 const MachineInstr &CallMI = *(--II);
5618 if (!CallMI.isCall() || !CallMI.getOperand(i: 0).isGlobal())
5619 return IsExtendPair;
5620
5621 const Function *CalleeFn =
5622 dyn_cast_if_present<Function>(Val: CallMI.getOperand(i: 0).getGlobal());
5623 if (!CalleeFn)
5624 return IsExtendPair;
5625 const IntegerType *IntTy = dyn_cast<IntegerType>(Val: CalleeFn->getReturnType());
5626 if (IntTy && IntTy->getBitWidth() <= 32) {
5627 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
5628 IsSExt |= Attrs.hasAttribute(Kind: Attribute::SExt);
5629 IsZExt |= Attrs.hasAttribute(Kind: Attribute::ZExt);
5630 return std::pair<bool, bool>(IsSExt, IsZExt);
5631 }
5632
5633 return IsExtendPair;
5634 }
5635
5636 // OR, XOR with 16-bit immediate does not change the upper 48 bits.
5637 // So, we track the operand register as we do for register copy.
5638 case PPC::ORI:
5639 case PPC::XORI:
5640 case PPC::ORI8:
5641 case PPC::XORI8: {
5642 Register SrcReg = MI->getOperand(i: 1).getReg();
5643 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5644 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5645 SrcExt.second || IsZExt);
5646 }
5647
5648 // OR, XOR with shifted 16-bit immediate does not change the upper
5649 // 32 bits. So, we track the operand register for zero extension.
5650 // For sign extension when the MSB of the immediate is zero, we also
5651 // track the operand register since the upper 33 bits are unchanged.
5652 case PPC::ORIS:
5653 case PPC::XORIS:
5654 case PPC::ORIS8:
5655 case PPC::XORIS8: {
5656 Register SrcReg = MI->getOperand(i: 1).getReg();
5657 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5658 uint16_t Imm = MI->getOperand(i: 2).getImm();
5659 if (Imm & 0x8000)
5660 return std::pair<bool, bool>(false, SrcExt.second || IsZExt);
5661 else
5662 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5663 SrcExt.second || IsZExt);
5664 }
5665
5666 // If all incoming values are sign-/zero-extended,
5667 // the output of OR, ISEL or PHI is also sign-/zero-extended.
5668 case PPC::OR:
5669 case PPC::OR8:
5670 case PPC::ISEL:
5671 case PPC::PHI: {
5672 if (BinOpDepth >= MAX_BINOP_DEPTH)
5673 return std::pair<bool, bool>(false, false);
5674
5675 // The input registers for PHI are operand 1, 3, ...
5676 // The input registers for others are operand 1 and 2.
5677 unsigned OperandEnd = 3, OperandStride = 1;
5678 if (MI->getOpcode() == PPC::PHI) {
5679 OperandEnd = MI->getNumOperands();
5680 OperandStride = 2;
5681 }
5682
5683 IsSExt = true;
5684 IsZExt = true;
5685 for (unsigned I = 1; I != OperandEnd; I += OperandStride) {
5686 if (!MI->getOperand(i: I).isReg())
5687 return std::pair<bool, bool>(false, false);
5688
5689 Register SrcReg = MI->getOperand(i: I).getReg();
5690 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth: BinOpDepth + 1, MRI);
5691 IsSExt &= SrcExt.first;
5692 IsZExt &= SrcExt.second;
5693 }
5694 return std::pair<bool, bool>(IsSExt, IsZExt);
5695 }
5696
5697 // If at least one of the incoming values of an AND is zero extended
5698 // then the output is also zero-extended. If both of the incoming values
5699 // are sign-extended then the output is also sign extended.
5700 case PPC::AND:
5701 case PPC::AND8: {
5702 if (BinOpDepth >= MAX_BINOP_DEPTH)
5703 return std::pair<bool, bool>(false, false);
5704
5705 Register SrcReg1 = MI->getOperand(i: 1).getReg();
5706 Register SrcReg2 = MI->getOperand(i: 2).getReg();
5707 auto Src1Ext = isSignOrZeroExtended(Reg: SrcReg1, BinOpDepth: BinOpDepth + 1, MRI);
5708 auto Src2Ext = isSignOrZeroExtended(Reg: SrcReg2, BinOpDepth: BinOpDepth + 1, MRI);
5709 return std::pair<bool, bool>(Src1Ext.first && Src2Ext.first,
5710 Src1Ext.second || Src2Ext.second);
5711 }
5712
5713 default:
5714 break;
5715 }
5716 return std::pair<bool, bool>(IsSExt, IsZExt);
5717}
5718
5719bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
5720 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
5721}
5722
5723namespace {
5724class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5725 MachineInstr *Loop, *EndLoop, *LoopCount;
5726 MachineFunction *MF;
5727 const TargetInstrInfo *TII;
5728 int64_t TripCount;
5729
5730public:
5731 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
5732 MachineInstr *LoopCount)
5733 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
5734 MF(Loop->getParent()->getParent()),
5735 TII(MF->getSubtarget().getInstrInfo()) {
5736 // Inspect the Loop instruction up-front, as it may be deleted when we call
5737 // createTripCountGreaterCondition.
5738 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
5739 TripCount = LoopCount->getOperand(i: 1).getImm();
5740 else
5741 TripCount = -1;
5742 }
5743
5744 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5745 // Only ignore the terminator.
5746 return MI == EndLoop;
5747 }
5748
5749 std::optional<bool> createTripCountGreaterCondition(
5750 int TC, MachineBasicBlock &MBB,
5751 SmallVectorImpl<MachineOperand> &Cond) override {
5752 if (TripCount == -1) {
5753 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5754 // so we don't need to generate any thing here.
5755 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
5756 Cond.push_back(Elt: MachineOperand::CreateReg(
5757 Reg: MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
5758 isDef: true));
5759 return {};
5760 }
5761
5762 return TripCount > TC;
5763 }
5764
5765 void setPreheader(MachineBasicBlock *NewPreheader) override {
5766 // Do nothing. We want the LOOP setup instruction to stay in the *old*
5767 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
5768 }
5769
5770 void adjustTripCount(int TripCountAdjust) override {
5771 // If the loop trip count is a compile-time value, then just change the
5772 // value.
5773 if (LoopCount->getOpcode() == PPC::LI8 ||
5774 LoopCount->getOpcode() == PPC::LI) {
5775 int64_t TripCount = LoopCount->getOperand(i: 1).getImm() + TripCountAdjust;
5776 LoopCount->getOperand(i: 1).setImm(TripCount);
5777 return;
5778 }
5779
5780 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5781 // so we don't need to generate any thing here.
5782 }
5783
5784 void disposed(LiveIntervals *LIS) override {
5785 if (LIS) {
5786 LIS->RemoveMachineInstrFromMaps(MI&: *Loop);
5787 LIS->RemoveMachineInstrFromMaps(MI&: *LoopCount);
5788 }
5789 Loop->eraseFromParent();
5790 // Ensure the loop setup instruction is deleted too.
5791 LoopCount->eraseFromParent();
5792 }
5793};
5794} // namespace
5795
5796std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5797PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5798 // We really "analyze" only hardware loops right now.
5799 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
5800 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
5801 if (Preheader == LoopBB)
5802 Preheader = *std::next(x: LoopBB->pred_begin());
5803 MachineFunction *MF = Preheader->getParent();
5804
5805 if (I != LoopBB->end() && isBDNZ(Opcode: I->getOpcode())) {
5806 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5807 if (MachineInstr *LoopInst = findLoopInstr(PreHeader&: *Preheader, Visited)) {
5808 Register LoopCountReg = LoopInst->getOperand(i: 0).getReg();
5809 MachineRegisterInfo &MRI = MF->getRegInfo();
5810 MachineInstr *LoopCount = MRI.getUniqueVRegDef(Reg: LoopCountReg);
5811 return std::make_unique<PPCPipelinerLoopInfo>(args&: LoopInst, args: &*I, args&: LoopCount);
5812 }
5813 }
5814 return nullptr;
5815}
5816
5817MachineInstr *PPCInstrInfo::findLoopInstr(
5818 MachineBasicBlock &PreHeader,
5819 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
5820
5821 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
5822
5823 // The loop set-up instruction should be in preheader
5824 for (auto &I : PreHeader.instrs())
5825 if (I.getOpcode() == LOOPi)
5826 return &I;
5827 return nullptr;
5828}
5829
5830// Return true if get the base operand, byte offset of an instruction and the
5831// memory width. Width is the size of memory that is being loaded/stored.
5832bool PPCInstrInfo::getMemOperandWithOffsetWidth(
5833 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
5834 LocationSize &Width, const TargetRegisterInfo *TRI) const {
5835 if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
5836 return false;
5837
5838 // Handle only loads/stores with base register followed by immediate offset.
5839 if (!LdSt.getOperand(i: 1).isImm() ||
5840 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()))
5841 return false;
5842
5843 if (!LdSt.hasOneMemOperand())
5844 return false;
5845
5846 Width = (*LdSt.memoperands_begin())->getSize();
5847 Offset = LdSt.getOperand(i: 1).getImm();
5848 BaseReg = &LdSt.getOperand(i: 2);
5849 return true;
5850}
5851
5852bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
5853 const MachineInstr &MIa, const MachineInstr &MIb) const {
5854 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
5855 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
5856
5857 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
5858 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
5859 return false;
5860
5861 // Retrieve the base register, offset from the base register and width. Width
5862 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
5863 // base registers are identical, and the offset of a lower memory access +
5864 // the width doesn't overlap the offset of a higher memory access,
5865 // then the memory accesses are different.
5866 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
5867 int64_t OffsetA = 0, OffsetB = 0;
5868 LocationSize WidthA = LocationSize::precise(Value: 0),
5869 WidthB = LocationSize::precise(Value: 0);
5870 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA, TRI: &RI) &&
5871 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB, TRI: &RI)) {
5872 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
5873 int LowOffset = std::min(a: OffsetA, b: OffsetB);
5874 int HighOffset = std::max(a: OffsetA, b: OffsetB);
5875 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
5876 if (LowWidth.hasValue() &&
5877 LowOffset + (int)LowWidth.getValue() <= HighOffset)
5878 return true;
5879 }
5880 }
5881 return false;
5882}
5883
5884// Expands LWAT_CSNE_PSEUDO/LDAT_CSNE_PSEUDO post register allocation.
5885// lwat/ldat FC=16 requires 3 consecutive registers. X8/X9/X10 are
5886// hardcoded post-RA to satisfy this constraint without a dedicated
5887// register class.
5888bool PPCInstrInfo::expandAMOCSNEPseudo(MachineInstr &MI) const {
5889 MachineBasicBlock &MBB = *MI.getParent();
5890 DebugLoc DL = MI.getDebugLoc();
5891 bool IsLDAT = MI.getOpcode() == PPC::LDAT_CSNE_PSEUDO;
5892
5893 Register DstReg = MI.getOperand(i: 0).getReg();
5894 Register PtrReg = MI.getOperand(i: 1).getReg();
5895
5896 Register ScratchReg = PtrReg;
5897 if (PtrReg == PPC::X8 || PtrReg == PPC::X9 || PtrReg == PPC::X10) {
5898 // If ptr is in X8/X9/X10, use $dst as scratch to move ptr away from
5899 // X8/X9/X10 since lwat FC=16 always writes its result to X8. After lwat
5900 // copy X8 into $dst.
5901 Register DstReg64 = IsLDAT ? DstReg
5902 : Register(RI.getMatchingSuperReg(
5903 Reg: DstReg, SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass));
5904 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DstReg64).addReg(RegNo: PtrReg).addReg(RegNo: PtrReg);
5905 ScratchReg = DstReg64;
5906 }
5907
5908 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: IsLDAT ? PPC::LDAT_CSNE : PPC::LWAT_CSNE), DestReg: PPC::X8)
5909 .addReg(RegNo: ScratchReg)
5910 .addReg(RegNo: PPC::X9, Flags: RegState::Implicit)
5911 .addReg(RegNo: PPC::X10, Flags: RegState::Implicit);
5912
5913 if (DstReg != (IsLDAT ? PPC::X8 : PPC::R8)) {
5914 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: IsLDAT ? PPC::OR8 : PPC::OR), DestReg: DstReg)
5915 .addReg(RegNo: IsLDAT ? PPC::X8 : PPC::R8)
5916 .addReg(RegNo: IsLDAT ? PPC::X8 : PPC::R8);
5917 }
5918 MI.eraseFromParent();
5919 return true;
5920}
5921