1//===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the PowerPC implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCInstrInfo.h"
14#include "MCTargetDesc/PPCPredicates.h"
15#include "PPC.h"
16#include "PPCHazardRecognizers.h"
17#include "PPCInstrBuilder.h"
18#include "PPCMachineFunctionInfo.h"
19#include "PPCTargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/CodeGen/LiveIntervals.h"
23#include "llvm/CodeGen/LivePhysRegs.h"
24#include "llvm/CodeGen/MachineCombinerPattern.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/PseudoSourceValue.h"
31#include "llvm/CodeGen/RegisterClassInfo.h"
32#include "llvm/CodeGen/RegisterPressure.h"
33#include "llvm/CodeGen/RegisterScavenging.h"
34#include "llvm/CodeGen/ScheduleDAG.h"
35#include "llvm/CodeGen/SlotIndexes.h"
36#include "llvm/CodeGen/StackMaps.h"
37#include "llvm/IR/Module.h"
38#include "llvm/MC/MCInst.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Support/raw_ostream.h"
44
45using namespace llvm;
46
47#define DEBUG_TYPE "ppc-instr-info"
48
49#define GET_INSTRMAP_INFO
50#define GET_INSTRINFO_CTOR_DTOR
51#include "PPCGenInstrInfo.inc"
52
53STATISTIC(NumStoreSPILLVSRRCAsVec,
54 "Number of spillvsrrc spilled to stack as vec");
55STATISTIC(NumStoreSPILLVSRRCAsGpr,
56 "Number of spillvsrrc spilled to stack as gpr");
57STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
58STATISTIC(CmpIselsConverted,
59 "Number of ISELs that depend on comparison of constants converted");
60STATISTIC(MissedConvertibleImmediateInstrs,
61 "Number of compare-immediate instructions fed by constants");
62STATISTIC(NumRcRotatesConvertedToRcAnd,
63 "Number of record-form rotates converted to record-form andi");
64
65static cl::opt<bool>
66 EnableCTRLoopAnal("ppc-ctr-loop-analysis",
67 cl::desc("Enable analysis for CTR loops"), cl::init(Val: true),
68 cl::Hidden);
69
70static cl::opt<bool>
71 EnableCmpOpt("ppc-cmp-opt",
72 cl::desc("Enable compare instruction optimization"),
73 cl::init(Val: true), cl::Hidden);
74
75static cl::opt<bool> VSXSelfCopyCrash(
76 "ppc-crash-on-vsx-self-copy",
77 cl::desc(
78 "Causes the backend to crash instead of generating a nop VSX copy"),
79 cl::Hidden);
80
81static cl::opt<bool>
82UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
83 cl::desc("Use the old (incorrect) instruction latency calculation"));
84
85static cl::opt<float>
86 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(Val: 1.5),
87 cl::desc("register pressure factor for the transformations."));
88
89static cl::opt<bool> EnableFMARegPressureReduction(
90 "ppc-fma-rp-reduction", cl::Hidden, cl::init(Val: true),
91 cl::desc("enable register pressure reduce in machine combiner pass."));
92
93// Pin the vtable to this file.
94void PPCInstrInfo::anchor() {}
95
96PPCInstrInfo::PPCInstrInfo(const PPCSubtarget &STI)
97 : PPCGenInstrInfo(STI, RI, PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
98 /* CatchRetOpcode */ -1,
99 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
100 Subtarget(STI), RI(STI.getTargetMachine()) {}
101
102const TargetRegisterClass *PPCInstrInfo::getInlineAsmMemoryOperandRegClass(
103 InlineAsm::ConstraintCode C) const {
104 return Subtarget.isPPC64() ? &PPC::G8RC_NOX0RegClass
105 : &PPC::GPRC_NOR0RegClass;
106}
107
108/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
109/// this target when scheduling the DAG.
110ScheduleHazardRecognizer *
111PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
112 const ScheduleDAG *DAG) const {
113 unsigned Directive =
114 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
115 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
116 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
117 const InstrItineraryData *II =
118 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
119 return new ScoreboardHazardRecognizer(II, DAG);
120 }
121
122 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
123}
124
125/// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
126/// to use for this target when scheduling the DAG.
127ScheduleHazardRecognizer *
128PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
129 const ScheduleDAG *DAG) const {
130 unsigned Directive =
131 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
132
133 // FIXME: Leaving this as-is until we have POWER9 scheduling info
134 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
135 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
136
137 // Most subtargets use a PPC970 recognizer.
138 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
139 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
140 assert(DAG->TII && "No InstrInfo?");
141
142 return new PPCHazardRecognizer970(*DAG);
143 }
144
145 return new ScoreboardHazardRecognizer(II, DAG);
146}
147
148unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
149 const MachineInstr &MI,
150 unsigned *PredCost) const {
151 if (!ItinData || UseOldLatencyCalc)
152 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
153
154 // The default implementation of getInstrLatency calls getStageLatency, but
155 // getStageLatency does not do the right thing for us. While we have
156 // itinerary, most cores are fully pipelined, and so the itineraries only
157 // express the first part of the pipeline, not every stage. Instead, we need
158 // to use the listed output operand cycle number (using operand 0 here, which
159 // is an output).
160
161 unsigned Latency = 1;
162 unsigned DefClass = MI.getDesc().getSchedClass();
163 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
164 const MachineOperand &MO = MI.getOperand(i);
165 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
166 continue;
167
168 std::optional<unsigned> Cycle = ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: i);
169 if (!Cycle)
170 continue;
171
172 Latency = std::max(a: Latency, b: *Cycle);
173 }
174
175 return Latency;
176}
177
178std::optional<unsigned> PPCInstrInfo::getOperandLatency(
179 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
180 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
181 std::optional<unsigned> Latency = PPCGenInstrInfo::getOperandLatency(
182 ItinData, DefMI, DefIdx, UseMI, UseIdx);
183
184 if (!DefMI.getParent())
185 return Latency;
186
187 const MachineOperand &DefMO = DefMI.getOperand(i: DefIdx);
188 Register Reg = DefMO.getReg();
189
190 bool IsRegCR;
191 if (Reg.isVirtual()) {
192 const MachineRegisterInfo *MRI =
193 &DefMI.getParent()->getParent()->getRegInfo();
194 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(RC: &PPC::CRRCRegClass) ||
195 MRI->getRegClass(Reg)->hasSuperClassEq(RC: &PPC::CRBITRCRegClass);
196 } else {
197 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
198 PPC::CRBITRCRegClass.contains(Reg);
199 }
200
201 if (UseMI.isBranch() && IsRegCR) {
202 if (!Latency)
203 Latency = getInstrLatency(ItinData, MI: DefMI);
204
205 // On some cores, there is an additional delay between writing to a condition
206 // register, and using it from a branch.
207 unsigned Directive = Subtarget.getCPUDirective();
208 switch (Directive) {
209 default: break;
210 case PPC::DIR_7400:
211 case PPC::DIR_750:
212 case PPC::DIR_970:
213 case PPC::DIR_E5500:
214 case PPC::DIR_PWR4:
215 case PPC::DIR_PWR5:
216 case PPC::DIR_PWR5X:
217 case PPC::DIR_PWR6:
218 case PPC::DIR_PWR6X:
219 case PPC::DIR_PWR7:
220 case PPC::DIR_PWR8:
221 // FIXME: Is this needed for POWER9?
222 Latency = *Latency + 2;
223 break;
224 }
225 }
226
227 return Latency;
228}
229
230void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
231 uint32_t Flags) const {
232 MI.setFlags(Flags);
233 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
234 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
235 MI.clearFlag(Flag: MachineInstr::MIFlag::IsExact);
236}
237
238// This function does not list all associative and commutative operations, but
239// only those worth feeding through the machine combiner in an attempt to
240// reduce the critical path. Mostly, this means floating-point operations,
241// because they have high latencies(>=5) (compared to other operations, such as
242// and/or, which are also associative and commutative, but have low latencies).
243bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
244 bool Invert) const {
245 if (Invert)
246 return false;
247 switch (Inst.getOpcode()) {
248 // Floating point:
249 // FP Add:
250 case PPC::FADD:
251 case PPC::FADDS:
252 // FP Multiply:
253 case PPC::FMUL:
254 case PPC::FMULS:
255 // Altivec Add:
256 case PPC::VADDFP:
257 // VSX Add:
258 case PPC::XSADDDP:
259 case PPC::XVADDDP:
260 case PPC::XVADDSP:
261 case PPC::XSADDSP:
262 // VSX Multiply:
263 case PPC::XSMULDP:
264 case PPC::XVMULDP:
265 case PPC::XVMULSP:
266 case PPC::XSMULSP:
267 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
268 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
269 // Fixed point:
270 // Multiply:
271 case PPC::MULHD:
272 case PPC::MULLD:
273 case PPC::MULHW:
274 case PPC::MULLW:
275 return true;
276 default:
277 return false;
278 }
279}
280
281#define InfoArrayIdxFMAInst 0
282#define InfoArrayIdxFAddInst 1
283#define InfoArrayIdxFMULInst 2
284#define InfoArrayIdxAddOpIdx 3
285#define InfoArrayIdxMULOpIdx 4
286#define InfoArrayIdxFSubInst 5
287// Array keeps info for FMA instructions:
288// Index 0(InfoArrayIdxFMAInst): FMA instruction;
289// Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
290// Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
291// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
292// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
293// second MUL operand index is plus 1;
294// Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
295static const uint16_t FMAOpIdxInfo[][6] = {
296 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
297 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
298 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
299 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
300 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
301 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
302 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
303
304// Check if an opcode is a FMA instruction. If it is, return the index in array
305// FMAOpIdxInfo. Otherwise, return -1.
306int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
307 for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
308 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
309 return I;
310 return -1;
311}
312
313// On PowerPC target, we have two kinds of patterns related to FMA:
314// 1: Improve ILP.
315// Try to reassociate FMA chains like below:
316//
317// Pattern 1:
318// A = FADD X, Y (Leaf)
319// B = FMA A, M21, M22 (Prev)
320// C = FMA B, M31, M32 (Root)
321// -->
322// A = FMA X, M21, M22
323// B = FMA Y, M31, M32
324// C = FADD A, B
325//
326// Pattern 2:
327// A = FMA X, M11, M12 (Leaf)
328// B = FMA A, M21, M22 (Prev)
329// C = FMA B, M31, M32 (Root)
330// -->
331// A = FMUL M11, M12
332// B = FMA X, M21, M22
333// D = FMA A, M31, M32
334// C = FADD B, D
335//
336// breaking the dependency between A and B, allowing FMA to be executed in
337// parallel (or back-to-back in a pipeline) instead of depending on each other.
338//
339// 2: Reduce register pressure.
340// Try to reassociate FMA with FSUB and a constant like below:
341// C is a floating point const.
342//
343// Pattern 1:
344// A = FSUB X, Y (Leaf)
345// D = FMA B, C, A (Root)
346// -->
347// A = FMA B, Y, -C
348// D = FMA A, X, C
349//
350// Pattern 2:
351// A = FSUB X, Y (Leaf)
352// D = FMA B, A, C (Root)
353// -->
354// A = FMA B, Y, -C
355// D = FMA A, X, C
356//
357// Before the transformation, A must be assigned with different hardware
358// register with D. After the transformation, A and D must be assigned with
359// same hardware register due to TIE attribute of FMA instructions.
360//
361bool PPCInstrInfo::getFMAPatterns(MachineInstr &Root,
362 SmallVectorImpl<unsigned> &Patterns,
363 bool DoRegPressureReduce) const {
364 MachineBasicBlock *MBB = Root.getParent();
365 const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
366
367 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
368 for (const auto &MO : Instr.explicit_operands())
369 if (!(MO.isReg() && MO.getReg().isVirtual()))
370 return false;
371 return true;
372 };
373
374 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
375 unsigned OpType) {
376 if (Instr.getOpcode() !=
377 FMAOpIdxInfo[getFMAOpIdxInfo(Opcode: Root.getOpcode())][OpType])
378 return false;
379
380 // Instruction can be reassociated.
381 // fast math flags may prohibit reassociation.
382 if (!(Instr.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
383 Instr.getFlag(Flag: MachineInstr::MIFlag::FmNsz)))
384 return false;
385
386 // Instruction operands are virtual registers for reassociation.
387 if (!IsAllOpsVirtualReg(Instr))
388 return false;
389
390 // For register pressure reassociation, the FSub must have only one use as
391 // we want to delete the sub to save its def.
392 if (OpType == InfoArrayIdxFSubInst &&
393 !MRI->hasOneNonDBGUse(RegNo: Instr.getOperand(i: 0).getReg()))
394 return false;
395
396 return true;
397 };
398
399 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
400 int16_t &MulOpIdx, bool IsLeaf) {
401 int16_t Idx = getFMAOpIdxInfo(Opcode: Instr.getOpcode());
402 if (Idx < 0)
403 return false;
404
405 // Instruction can be reassociated.
406 // fast math flags may prohibit reassociation.
407 if (!(Instr.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
408 Instr.getFlag(Flag: MachineInstr::MIFlag::FmNsz)))
409 return false;
410
411 // Instruction operands are virtual registers for reassociation.
412 if (!IsAllOpsVirtualReg(Instr))
413 return false;
414
415 MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
416 if (IsLeaf)
417 return true;
418
419 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
420
421 const MachineOperand &OpAdd = Instr.getOperand(i: AddOpIdx);
422 MachineInstr *MIAdd = MRI->getUniqueVRegDef(Reg: OpAdd.getReg());
423 // If 'add' operand's def is not in current block, don't do ILP related opt.
424 if (!MIAdd || MIAdd->getParent() != MBB)
425 return false;
426
427 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
428 // as this fma will be changed later.
429 return MRI->hasOneNonDBGUse(RegNo: OpAdd.getReg());
430 };
431
432 int16_t AddOpIdx = -1;
433 int16_t MulOpIdx = -1;
434
435 bool IsUsedOnceL = false;
436 bool IsUsedOnceR = false;
437 MachineInstr *MULInstrL = nullptr;
438 MachineInstr *MULInstrR = nullptr;
439
440 auto IsRPReductionCandidate = [&]() {
441 // Currently, we only support float and double.
442 // FIXME: add support for other types.
443 unsigned Opcode = Root.getOpcode();
444 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
445 return false;
446
447 // Root must be a valid FMA like instruction.
448 // Treat it as leaf as we don't care its add operand.
449 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
450 assert((MulOpIdx >= 0) && "mul operand index not right!");
451 Register MULRegL = RI.lookThruSingleUseCopyChain(
452 SrcReg: Root.getOperand(i: MulOpIdx).getReg(), MRI);
453 Register MULRegR = RI.lookThruSingleUseCopyChain(
454 SrcReg: Root.getOperand(i: MulOpIdx + 1).getReg(), MRI);
455 if (!MULRegL && !MULRegR)
456 return false;
457
458 if (MULRegL && !MULRegR) {
459 MULRegR =
460 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: MulOpIdx + 1).getReg(), MRI);
461 IsUsedOnceL = true;
462 } else if (!MULRegL && MULRegR) {
463 MULRegL = RI.lookThruCopyLike(SrcReg: Root.getOperand(i: MulOpIdx).getReg(), MRI);
464 IsUsedOnceR = true;
465 } else {
466 IsUsedOnceL = true;
467 IsUsedOnceR = true;
468 }
469
470 if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
471 return false;
472
473 MULInstrL = MRI->getVRegDef(Reg: MULRegL);
474 MULInstrR = MRI->getVRegDef(Reg: MULRegR);
475 return MULInstrL && MULInstrR;
476 }
477 return false;
478 };
479
480 // Register pressure fma reassociation patterns.
481 if (DoRegPressureReduce && IsRPReductionCandidate()) {
482 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
483 // Register pressure pattern 1
484 if (isLoadFromConstantPool(I: MULInstrL) && IsUsedOnceR &&
485 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
486 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
487 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_BCA);
488 return true;
489 }
490
491 // Register pressure pattern 2
492 if ((isLoadFromConstantPool(I: MULInstrR) && IsUsedOnceL &&
493 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
494 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
495 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_BAC);
496 return true;
497 }
498 }
499
500 // ILP fma reassociation patterns.
501 // Root must be a valid FMA like instruction.
502 AddOpIdx = -1;
503 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
504 return false;
505
506 assert((AddOpIdx >= 0) && "add operand index not right!");
507
508 Register RegB = Root.getOperand(i: AddOpIdx).getReg();
509 MachineInstr *Prev = MRI->getUniqueVRegDef(Reg: RegB);
510
511 // Prev must be a valid FMA like instruction.
512 AddOpIdx = -1;
513 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
514 return false;
515
516 assert((AddOpIdx >= 0) && "add operand index not right!");
517
518 Register RegA = Prev->getOperand(i: AddOpIdx).getReg();
519 MachineInstr *Leaf = MRI->getUniqueVRegDef(Reg: RegA);
520 AddOpIdx = -1;
521 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
522 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
523 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
524 return true;
525 }
526 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
527 Patterns.push_back(Elt: PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM);
528 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
529 return true;
530 }
531 return false;
532}
533
534void PPCInstrInfo::finalizeInsInstrs(
535 MachineInstr &Root, unsigned &Pattern,
536 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
537 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
538
539 MachineFunction *MF = Root.getMF();
540 MachineRegisterInfo *MRI = &MF->getRegInfo();
541 MachineConstantPool *MCP = MF->getConstantPool();
542
543 int16_t Idx = getFMAOpIdxInfo(Opcode: Root.getOpcode());
544 if (Idx < 0)
545 return;
546
547 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
548
549 // For now we only need to fix up placeholder for register pressure reduce
550 // patterns.
551 Register ConstReg = 0;
552 switch (Pattern) {
553 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
554 ConstReg =
555 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx).getReg(), MRI);
556 break;
557 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
558 ConstReg =
559 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx + 1).getReg(), MRI);
560 break;
561 default:
562 // Not register pressure reduce patterns.
563 return;
564 }
565
566 MachineInstr *ConstDefInstr = MRI->getVRegDef(Reg: ConstReg);
567 // Get const value from const pool.
568 const Constant *C = getConstantFromConstantPool(I: ConstDefInstr);
569 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
570
571 // Get negative fp const.
572 APFloat F1((dyn_cast<ConstantFP>(Val: C))->getValueAPF());
573 F1.changeSign();
574 Constant *NegC = ConstantFP::get(Context&: dyn_cast<ConstantFP>(Val: C)->getContext(), V: F1);
575 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Ty: C->getType());
576
577 // Put negative fp const into constant pool.
578 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(C: NegC, Alignment);
579
580 MachineOperand *Placeholder = nullptr;
581 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
582 for (auto *Inst : InsInstrs) {
583 for (MachineOperand &Operand : Inst->explicit_operands()) {
584 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
585 if (Operand.getReg() == PPC::ZERO8) {
586 Placeholder = &Operand;
587 break;
588 }
589 }
590 }
591
592 assert(Placeholder && "Placeholder does not exist!");
593
594 // Generate instructions to load the const fp from constant pool.
595 // We only support PPC64 and medium code model.
596 Register LoadNewConst =
597 generateLoadForNewConst(Idx: ConstPoolIdx, MI: &Root, Ty: C->getType(), InsInstrs);
598
599 // Fill the placeholder with the new load from constant pool.
600 Placeholder->setReg(LoadNewConst);
601}
602
603bool PPCInstrInfo::shouldReduceRegisterPressure(
604 const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
605
606 if (!EnableFMARegPressureReduction)
607 return false;
608
609 // Currently, we only enable register pressure reducing in machine combiner
610 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
611 // support.
612 //
613 // So we need following instructions to access a TOC entry:
614 //
615 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
616 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
617 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
618 //
619 // FIXME: add more supported targets, like Small and Large code model, PPC32,
620 // AIX.
621 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
622 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
623 return false;
624
625 const MachineFunction *MF = MBB->getParent();
626 const MachineRegisterInfo *MRI = &MF->getRegInfo();
627
628 auto GetMBBPressure =
629 [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
630 RegionPressure Pressure;
631 RegPressureTracker RPTracker(Pressure);
632
633 // Initialize the register pressure tracker.
634 RPTracker.init(mf: MBB->getParent(), rci: RegClassInfo, lis: nullptr, mbb: MBB, pos: MBB->end(),
635 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
636
637 for (const auto &MI : reverse(C: *MBB)) {
638 if (MI.isDebugValue() || MI.isDebugLabel())
639 continue;
640 RegisterOperands RegOpers;
641 RegOpers.collect(MI, TRI: RI, MRI: *MRI, TrackLaneMasks: false, IgnoreDead: false);
642 RPTracker.recedeSkipDebugValues();
643 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
644 RPTracker.recede(RegOpers);
645 }
646
647 // Close the RPTracker to finalize live ins.
648 RPTracker.closeRegion();
649
650 return RPTracker.getPressure().MaxSetPressure;
651 };
652
653 // For now we only care about float and double type fma.
654 unsigned VSSRCLimit =
655 RegClassInfo->getRegPressureSetLimit(Idx: PPC::RegisterPressureSets::VSSRC);
656
657 // Only reduce register pressure when pressure is high.
658 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
659 (float)VSSRCLimit * FMARPFactor;
660}
661
662bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
663 // I has only one memory operand which is load from constant pool.
664 if (!I->hasOneMemOperand())
665 return false;
666
667 MachineMemOperand *Op = I->memoperands()[0];
668 return Op->isLoad() && Op->getPseudoValue() &&
669 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
670}
671
672Register PPCInstrInfo::generateLoadForNewConst(
673 unsigned Idx, MachineInstr *MI, Type *Ty,
674 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
675 // Now we only support PPC64, Medium code model and P9 with vector.
676 // We have immutable pattern to access const pool. See function
677 // shouldReduceRegisterPressure.
678 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
679 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
680 "Target not supported!\n");
681
682 MachineFunction *MF = MI->getMF();
683 MachineRegisterInfo *MRI = &MF->getRegInfo();
684
685 // Generate ADDIStocHA8
686 Register VReg1 = MRI->createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
687 MachineInstrBuilder TOCOffset =
688 BuildMI(MF&: *MF, MIMD: MI->getDebugLoc(), MCID: get(Opcode: PPC::ADDIStocHA8), DestReg: VReg1)
689 .addReg(RegNo: PPC::X2)
690 .addConstantPoolIndex(Idx);
691
692 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
693 "Only float and double are supported!");
694
695 unsigned LoadOpcode;
696 // Should be float type or double type.
697 if (Ty->isFloatTy())
698 LoadOpcode = PPC::DFLOADf32;
699 else
700 LoadOpcode = PPC::DFLOADf64;
701
702 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MI->getOperand(i: 0).getReg());
703 Register VReg2 = MRI->createVirtualRegister(RegClass: RC);
704 MachineMemOperand *MMO = MF->getMachineMemOperand(
705 PtrInfo: MachinePointerInfo::getConstantPool(MF&: *MF), F: MachineMemOperand::MOLoad,
706 Size: Ty->getScalarSizeInBits() / 8, BaseAlignment: MF->getDataLayout().getPrefTypeAlign(Ty));
707
708 // Generate Load from constant pool.
709 MachineInstrBuilder Load =
710 BuildMI(MF&: *MF, MIMD: MI->getDebugLoc(), MCID: get(Opcode: LoadOpcode), DestReg: VReg2)
711 .addConstantPoolIndex(Idx)
712 .addReg(RegNo: VReg1, Flags: getKillRegState(B: true))
713 .addMemOperand(MMO);
714
715 Load->getOperand(i: 1).setTargetFlags(PPCII::MO_TOC_LO);
716
717 // Insert the toc load instructions into InsInstrs.
718 InsInstrs.insert(I: InsInstrs.begin(), Elt: Load);
719 InsInstrs.insert(I: InsInstrs.begin(), Elt: TOCOffset);
720 return VReg2;
721}
722
723// This function returns the const value in constant pool if the \p I is a load
724// from constant pool.
725const Constant *
726PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
727 MachineFunction *MF = I->getMF();
728 MachineRegisterInfo *MRI = &MF->getRegInfo();
729 MachineConstantPool *MCP = MF->getConstantPool();
730 assert(I->mayLoad() && "Should be a load instruction.\n");
731 for (auto MO : I->uses()) {
732 if (!MO.isReg())
733 continue;
734 Register Reg = MO.getReg();
735 if (Reg == 0 || !Reg.isVirtual())
736 continue;
737 // Find the toc address.
738 MachineInstr *DefMI = MRI->getVRegDef(Reg);
739 for (auto MO2 : DefMI->uses())
740 if (MO2.isCPI())
741 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
742 }
743 return nullptr;
744}
745
746CombinerObjective PPCInstrInfo::getCombinerObjective(unsigned Pattern) const {
747 switch (Pattern) {
748 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
749 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
750 return CombinerObjective::MustReduceDepth;
751 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
752 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
753 return CombinerObjective::MustReduceRegisterPressure;
754 default:
755 return TargetInstrInfo::getCombinerObjective(Pattern);
756 }
757}
758
759bool PPCInstrInfo::getMachineCombinerPatterns(
760 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
761 bool DoRegPressureReduce) const {
762 // Using the machine combiner in this way is potentially expensive, so
763 // restrict to when aggressive optimizations are desired.
764 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOptLevel::Aggressive)
765 return false;
766
767 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
768 return true;
769
770 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
771 DoRegPressureReduce);
772}
773
774void PPCInstrInfo::genAlternativeCodeSequence(
775 MachineInstr &Root, unsigned Pattern,
776 SmallVectorImpl<MachineInstr *> &InsInstrs,
777 SmallVectorImpl<MachineInstr *> &DelInstrs,
778 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
779 switch (Pattern) {
780 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
781 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
782 case PPCMachineCombinerPattern::REASSOC_XY_BCA:
783 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
784 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
785 break;
786 default:
787 // Reassociate default patterns.
788 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
789 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
790 break;
791 }
792}
793
794void PPCInstrInfo::reassociateFMA(
795 MachineInstr &Root, unsigned Pattern,
796 SmallVectorImpl<MachineInstr *> &InsInstrs,
797 SmallVectorImpl<MachineInstr *> &DelInstrs,
798 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
799 MachineFunction *MF = Root.getMF();
800 MachineRegisterInfo &MRI = MF->getRegInfo();
801 MachineOperand &OpC = Root.getOperand(i: 0);
802 Register RegC = OpC.getReg();
803 const TargetRegisterClass *RC = MRI.getRegClass(Reg: RegC);
804 MRI.constrainRegClass(Reg: RegC, RC);
805
806 unsigned FmaOp = Root.getOpcode();
807 int16_t Idx = getFMAOpIdxInfo(Opcode: FmaOp);
808 assert(Idx >= 0 && "Root must be a FMA instruction");
809
810 bool IsILPReassociate =
811 (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
812 (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
813
814 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
815 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
816
817 MachineInstr *Prev = nullptr;
818 MachineInstr *Leaf = nullptr;
819 switch (Pattern) {
820 default:
821 llvm_unreachable("not recognized pattern!");
822 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
823 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
824 Prev = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: AddOpIdx).getReg());
825 Leaf = MRI.getUniqueVRegDef(Reg: Prev->getOperand(i: AddOpIdx).getReg());
826 break;
827 case PPCMachineCombinerPattern::REASSOC_XY_BAC: {
828 Register MULReg =
829 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx).getReg(), MRI: &MRI);
830 Leaf = MRI.getVRegDef(Reg: MULReg);
831 break;
832 }
833 case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
834 Register MULReg =
835 RI.lookThruCopyLike(SrcReg: Root.getOperand(i: FirstMulOpIdx + 1).getReg(), MRI: &MRI);
836 Leaf = MRI.getVRegDef(Reg: MULReg);
837 break;
838 }
839 }
840
841 uint32_t IntersectedFlags = 0;
842 if (IsILPReassociate)
843 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
844 else
845 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
846
847 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
848 bool &KillFlag) {
849 Reg = Operand.getReg();
850 MRI.constrainRegClass(Reg, RC);
851 KillFlag = Operand.isKill();
852 };
853
854 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
855 Register &MulOp2, Register &AddOp,
856 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
857 bool &AddOpKillFlag) {
858 GetOperandInfo(Instr.getOperand(i: FirstMulOpIdx), MulOp1, MulOp1KillFlag);
859 GetOperandInfo(Instr.getOperand(i: FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
860 GetOperandInfo(Instr.getOperand(i: AddOpIdx), AddOp, AddOpKillFlag);
861 };
862
863 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
864 RegA21, RegB;
865 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
866 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
867 KillA11 = false, KillA21 = false, KillB = false;
868
869 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
870
871 if (IsILPReassociate)
872 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
873
874 if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
875 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
876 GetOperandInfo(Leaf->getOperand(i: AddOpIdx), RegX, KillX);
877 } else if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) {
878 GetOperandInfo(Leaf->getOperand(i: 1), RegX, KillX);
879 GetOperandInfo(Leaf->getOperand(i: 2), RegY, KillY);
880 } else {
881 // Get FSUB instruction info.
882 GetOperandInfo(Leaf->getOperand(i: 1), RegX, KillX);
883 GetOperandInfo(Leaf->getOperand(i: 2), RegY, KillY);
884 }
885
886 // Create new virtual registers for the new results instead of
887 // recycling legacy ones because the MachineCombiner's computation of the
888 // critical path requires a new register definition rather than an existing
889 // one.
890 // For register pressure reassociation, we only need create one virtual
891 // register for the new fma.
892 Register NewVRA = MRI.createVirtualRegister(RegClass: RC);
893 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRA, y: 0));
894
895 Register NewVRB = 0;
896 if (IsILPReassociate) {
897 NewVRB = MRI.createVirtualRegister(RegClass: RC);
898 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRB, y: 1));
899 }
900
901 Register NewVRD = 0;
902 if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
903 NewVRD = MRI.createVirtualRegister(RegClass: RC);
904 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVRD, y: 2));
905 }
906
907 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
908 Register RegMul1, bool KillRegMul1,
909 Register RegMul2, bool KillRegMul2) {
910 MI->getOperand(i: AddOpIdx).setReg(RegAdd);
911 MI->getOperand(i: AddOpIdx).setIsKill(KillAdd);
912 MI->getOperand(i: FirstMulOpIdx).setReg(RegMul1);
913 MI->getOperand(i: FirstMulOpIdx).setIsKill(KillRegMul1);
914 MI->getOperand(i: FirstMulOpIdx + 1).setReg(RegMul2);
915 MI->getOperand(i: FirstMulOpIdx + 1).setIsKill(KillRegMul2);
916 };
917
918 MachineInstrBuilder NewARegPressure, NewCRegPressure;
919 switch (Pattern) {
920 default:
921 llvm_unreachable("not recognized pattern!");
922 case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM: {
923 // Create new instructions for insertion.
924 MachineInstrBuilder MINewB =
925 BuildMI(MF&: *MF, MIMD: Prev->getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRB)
926 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
927 .addReg(RegNo: RegM21, Flags: getKillRegState(B: KillM21))
928 .addReg(RegNo: RegM22, Flags: getKillRegState(B: KillM22));
929 MachineInstrBuilder MINewA =
930 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRA)
931 .addReg(RegNo: RegY, Flags: getKillRegState(B: KillY))
932 .addReg(RegNo: RegM31, Flags: getKillRegState(B: KillM31))
933 .addReg(RegNo: RegM32, Flags: getKillRegState(B: KillM32));
934 // If AddOpIdx is not 1, adjust the order.
935 if (AddOpIdx != 1) {
936 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
937 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
938 }
939
940 MachineInstrBuilder MINewC =
941 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(),
942 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), DestReg: RegC)
943 .addReg(RegNo: NewVRB, Flags: getKillRegState(B: true))
944 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true));
945
946 // Update flags for newly created instructions.
947 setSpecialOperandAttr(MI&: *MINewA, Flags: IntersectedFlags);
948 setSpecialOperandAttr(MI&: *MINewB, Flags: IntersectedFlags);
949 setSpecialOperandAttr(MI&: *MINewC, Flags: IntersectedFlags);
950
951 // Record new instructions for insertion.
952 InsInstrs.push_back(Elt: MINewA);
953 InsInstrs.push_back(Elt: MINewB);
954 InsInstrs.push_back(Elt: MINewC);
955 break;
956 }
957 case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
958 assert(NewVRD && "new FMA register not created!");
959 // Create new instructions for insertion.
960 MachineInstrBuilder MINewA =
961 BuildMI(MF&: *MF, MIMD: Leaf->getDebugLoc(),
962 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), DestReg: NewVRA)
963 .addReg(RegNo: RegM11, Flags: getKillRegState(B: KillM11))
964 .addReg(RegNo: RegM12, Flags: getKillRegState(B: KillM12));
965 MachineInstrBuilder MINewB =
966 BuildMI(MF&: *MF, MIMD: Prev->getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRB)
967 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
968 .addReg(RegNo: RegM21, Flags: getKillRegState(B: KillM21))
969 .addReg(RegNo: RegM22, Flags: getKillRegState(B: KillM22));
970 MachineInstrBuilder MINewD =
971 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRD)
972 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true))
973 .addReg(RegNo: RegM31, Flags: getKillRegState(B: KillM31))
974 .addReg(RegNo: RegM32, Flags: getKillRegState(B: KillM32));
975 // If AddOpIdx is not 1, adjust the order.
976 if (AddOpIdx != 1) {
977 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
978 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
979 KillM32);
980 }
981
982 MachineInstrBuilder MINewC =
983 BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(),
984 MCID: get(Opcode: FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), DestReg: RegC)
985 .addReg(RegNo: NewVRB, Flags: getKillRegState(B: true))
986 .addReg(RegNo: NewVRD, Flags: getKillRegState(B: true));
987
988 // Update flags for newly created instructions.
989 setSpecialOperandAttr(MI&: *MINewA, Flags: IntersectedFlags);
990 setSpecialOperandAttr(MI&: *MINewB, Flags: IntersectedFlags);
991 setSpecialOperandAttr(MI&: *MINewD, Flags: IntersectedFlags);
992 setSpecialOperandAttr(MI&: *MINewC, Flags: IntersectedFlags);
993
994 // Record new instructions for insertion.
995 InsInstrs.push_back(Elt: MINewA);
996 InsInstrs.push_back(Elt: MINewB);
997 InsInstrs.push_back(Elt: MINewD);
998 InsInstrs.push_back(Elt: MINewC);
999 break;
1000 }
1001 case PPCMachineCombinerPattern::REASSOC_XY_BAC:
1002 case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
1003 Register VarReg;
1004 bool KillVarReg = false;
1005 if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_BCA) {
1006 VarReg = RegM31;
1007 KillVarReg = KillM31;
1008 } else {
1009 VarReg = RegM32;
1010 KillVarReg = KillM32;
1011 }
1012 // We don't want to get negative const from memory pool too early, as the
1013 // created entry will not be deleted even if it has no users. Since all
1014 // operand of Leaf and Root are virtual register, we use zero register
1015 // here as a placeholder. When the InsInstrs is selected in
1016 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1017 // with a virtual register which is a load from constant pool.
1018 NewARegPressure = BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: NewVRA)
1019 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegB))
1020 .addReg(RegNo: RegY, Flags: getKillRegState(B: KillY))
1021 .addReg(RegNo: PPC::ZERO8);
1022 NewCRegPressure = BuildMI(MF&: *MF, MIMD: Root.getDebugLoc(), MCID: get(Opcode: FmaOp), DestReg: RegC)
1023 .addReg(RegNo: NewVRA, Flags: getKillRegState(B: true))
1024 .addReg(RegNo: RegX, Flags: getKillRegState(B: KillX))
1025 .addReg(RegNo: VarReg, Flags: getKillRegState(B: KillVarReg));
1026 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1027 // both at index 1, no need to adjust.
1028 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1029 // the operand index here.
1030 break;
1031 }
1032 }
1033
1034 if (!IsILPReassociate) {
1035 setSpecialOperandAttr(MI&: *NewARegPressure, Flags: IntersectedFlags);
1036 setSpecialOperandAttr(MI&: *NewCRegPressure, Flags: IntersectedFlags);
1037
1038 InsInstrs.push_back(Elt: NewARegPressure);
1039 InsInstrs.push_back(Elt: NewCRegPressure);
1040 }
1041
1042 assert(!InsInstrs.empty() &&
1043 "Insertion instructions set should not be empty!");
1044
1045 // Record old instructions for deletion.
1046 DelInstrs.push_back(Elt: Leaf);
1047 if (IsILPReassociate)
1048 DelInstrs.push_back(Elt: Prev);
1049 DelInstrs.push_back(Elt: &Root);
1050}
1051
1052// Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1053bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1054 Register &SrcReg, Register &DstReg,
1055 unsigned &SubIdx) const {
1056 switch (MI.getOpcode()) {
1057 default: return false;
1058 case PPC::EXTSW:
1059 case PPC::EXTSW_32:
1060 case PPC::EXTSW_32_64:
1061 SrcReg = MI.getOperand(i: 1).getReg();
1062 DstReg = MI.getOperand(i: 0).getReg();
1063 SubIdx = PPC::sub_32;
1064 return true;
1065 }
1066}
1067
1068Register PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1069 int &FrameIndex) const {
1070 if (llvm::is_contained(Range: getLoadOpcodesForSpillArray(), Element: MI.getOpcode())) {
1071 // Check for the operands added by addFrameReference (the immediate is the
1072 // offset which defaults to 0).
1073 if (MI.getOperand(i: 1).isImm() && !MI.getOperand(i: 1).getImm() &&
1074 MI.getOperand(i: 2).isFI()) {
1075 FrameIndex = MI.getOperand(i: 2).getIndex();
1076 return MI.getOperand(i: 0).getReg();
1077 }
1078 }
1079 return 0;
1080}
1081
1082// For opcodes with the ReMaterializable flag set, this function is called to
1083// verify the instruction is really rematable.
1084bool PPCInstrInfo::isReMaterializableImpl(
1085 const MachineInstr &MI) const {
1086 switch (MI.getOpcode()) {
1087 default:
1088 // Let base implementaion decide.
1089 break;
1090 case PPC::LI:
1091 case PPC::LI8:
1092 case PPC::PLI:
1093 case PPC::PLI8:
1094 case PPC::LIS:
1095 case PPC::LIS8:
1096 case PPC::ADDIStocHA:
1097 case PPC::ADDIStocHA8:
1098 case PPC::ADDItocL:
1099 case PPC::ADDItocL8:
1100 case PPC::LOAD_STACK_GUARD:
1101 case PPC::PPCLdFixedAddr:
1102 case PPC::XXLXORz:
1103 case PPC::XXLXORspz:
1104 case PPC::XXLXORdpz:
1105 case PPC::XXLEQVOnes:
1106 case PPC::XXSPLTI32DX:
1107 case PPC::XXSPLTIW:
1108 case PPC::XXSPLTIDP:
1109 case PPC::V_SET0B:
1110 case PPC::V_SET0H:
1111 case PPC::V_SET0:
1112 case PPC::V_SETALLONESB:
1113 case PPC::V_SETALLONESH:
1114 case PPC::V_SETALLONES:
1115 case PPC::CRSET:
1116 case PPC::CRUNSET:
1117 case PPC::XXSETACCZ:
1118 case PPC::DMXXSETACCZ:
1119 return true;
1120 }
1121 return TargetInstrInfo::isReMaterializableImpl(MI);
1122}
1123
1124Register PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1125 int &FrameIndex) const {
1126 if (llvm::is_contained(Range: getStoreOpcodesForSpillArray(), Element: MI.getOpcode())) {
1127 if (MI.getOperand(i: 1).isImm() && !MI.getOperand(i: 1).getImm() &&
1128 MI.getOperand(i: 2).isFI()) {
1129 FrameIndex = MI.getOperand(i: 2).getIndex();
1130 return MI.getOperand(i: 0).getReg();
1131 }
1132 }
1133 return 0;
1134}
1135
1136MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1137 unsigned OpIdx1,
1138 unsigned OpIdx2) const {
1139 MachineFunction &MF = *MI.getParent()->getParent();
1140
1141 // Normal instructions can be commuted the obvious way.
1142 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1143 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1144 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1145 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1146 // changing the relative order of the mask operands might change what happens
1147 // to the high-bits of the mask (and, thus, the result).
1148
1149 // Cannot commute if it has a non-zero rotate count.
1150 if (MI.getOperand(i: 3).getImm() != 0)
1151 return nullptr;
1152
1153 // If we have a zero rotate count, we have:
1154 // M = mask(MB,ME)
1155 // Op0 = (Op1 & ~M) | (Op2 & M)
1156 // Change this to:
1157 // M = mask((ME+1)&31, (MB-1)&31)
1158 // Op0 = (Op2 & ~M) | (Op1 & M)
1159
1160 // Swap op1/op2
1161 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1162 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1163 Register Reg0 = MI.getOperand(i: 0).getReg();
1164 Register Reg1 = MI.getOperand(i: 1).getReg();
1165 Register Reg2 = MI.getOperand(i: 2).getReg();
1166 unsigned SubReg1 = MI.getOperand(i: 1).getSubReg();
1167 unsigned SubReg2 = MI.getOperand(i: 2).getSubReg();
1168 bool Reg1IsKill = MI.getOperand(i: 1).isKill();
1169 bool Reg2IsKill = MI.getOperand(i: 2).isKill();
1170 bool ChangeReg0 = false;
1171 // If machine instrs are no longer in two-address forms, update
1172 // destination register as well.
1173 if (Reg0 == Reg1) {
1174 // Must be two address instruction (i.e. op1 is tied to op0).
1175 assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
1176 "Expecting a two-address instruction!");
1177 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1178 Reg2IsKill = false;
1179 ChangeReg0 = true;
1180 }
1181
1182 // Masks.
1183 unsigned MB = MI.getOperand(i: 4).getImm();
1184 unsigned ME = MI.getOperand(i: 5).getImm();
1185
1186 // We can't commute a trivial mask (there is no way to represent an all-zero
1187 // mask).
1188 if (MB == 0 && ME == 31)
1189 return nullptr;
1190
1191 if (NewMI) {
1192 // Create a new instruction.
1193 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(i: 0).getReg();
1194 bool Reg0IsDead = MI.getOperand(i: 0).isDead();
1195 return BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: MI.getDesc())
1196 .addReg(RegNo: Reg0, Flags: RegState::Define | getDeadRegState(B: Reg0IsDead))
1197 .addReg(RegNo: Reg2, Flags: getKillRegState(B: Reg2IsKill))
1198 .addReg(RegNo: Reg1, Flags: getKillRegState(B: Reg1IsKill))
1199 .addImm(Val: (ME + 1) & 31)
1200 .addImm(Val: (MB - 1) & 31);
1201 }
1202
1203 if (ChangeReg0) {
1204 MI.getOperand(i: 0).setReg(Reg2);
1205 MI.getOperand(i: 0).setSubReg(SubReg2);
1206 }
1207 MI.getOperand(i: 2).setReg(Reg1);
1208 MI.getOperand(i: 1).setReg(Reg2);
1209 MI.getOperand(i: 2).setSubReg(SubReg1);
1210 MI.getOperand(i: 1).setSubReg(SubReg2);
1211 MI.getOperand(i: 2).setIsKill(Reg1IsKill);
1212 MI.getOperand(i: 1).setIsKill(Reg2IsKill);
1213
1214 // Swap the mask around.
1215 MI.getOperand(i: 4).setImm((ME + 1) & 31);
1216 MI.getOperand(i: 5).setImm((MB - 1) & 31);
1217 return &MI;
1218}
1219
1220bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1221 unsigned &SrcOpIdx1,
1222 unsigned &SrcOpIdx2) const {
1223 // For VSX A-Type FMA instructions, it is the first two operands that can be
1224 // commuted, however, because the non-encoded tied input operand is listed
1225 // first, the operands to swap are actually the second and third.
1226
1227 int AltOpc = PPC::getAltVSXFMAOpcode(Opcode: MI.getOpcode());
1228 if (AltOpc == -1)
1229 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1230
1231 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1232 // and SrcOpIdx2.
1233 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
1234}
1235
1236void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
1237 MachineBasicBlock::iterator MI) const {
1238 // This function is used for scheduling, and the nop wanted here is the type
1239 // that terminates dispatch groups on the POWER cores.
1240 unsigned Directive = Subtarget.getCPUDirective();
1241 unsigned Opcode;
1242 switch (Directive) {
1243 default: Opcode = PPC::NOP; break;
1244 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1245 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1246 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1247 // FIXME: Update when POWER9 scheduling model is ready.
1248 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1249 }
1250
1251 DebugLoc DL;
1252 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode));
1253}
1254
1255/// Return the noop instruction to use for a noop.
1256MCInst PPCInstrInfo::getNop() const {
1257 MCInst Nop;
1258 Nop.setOpcode(PPC::NOP);
1259 return Nop;
1260}
1261
1262// Branch analysis.
1263// Note: If the condition register is set to CTR or CTR8 then this is a
1264// BDNZ (imm == 1) or BDZ (imm == 0) branch.
1265bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1266 MachineBasicBlock *&TBB,
1267 MachineBasicBlock *&FBB,
1268 SmallVectorImpl<MachineOperand> &Cond,
1269 bool AllowModify) const {
1270 bool isPPC64 = Subtarget.isPPC64();
1271
1272 // If the block has no terminators, it just falls into the block after it.
1273 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1274 if (I == MBB.end())
1275 return false;
1276
1277 if (!isUnpredicatedTerminator(MI: *I))
1278 return false;
1279
1280 if (AllowModify) {
1281 // If the BB ends with an unconditional branch to the fallthrough BB,
1282 // we eliminate the branch instruction.
1283 if (I->getOpcode() == PPC::B &&
1284 MBB.isLayoutSuccessor(MBB: I->getOperand(i: 0).getMBB())) {
1285 I->eraseFromParent();
1286
1287 // We update iterator after deleting the last branch.
1288 I = MBB.getLastNonDebugInstr();
1289 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1290 return false;
1291 }
1292 }
1293
1294 // Get the last instruction in the block.
1295 MachineInstr &LastInst = *I;
1296
1297 // If there is only one terminator instruction, process it.
1298 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
1299 if (LastInst.getOpcode() == PPC::B) {
1300 if (!LastInst.getOperand(i: 0).isMBB())
1301 return true;
1302 TBB = LastInst.getOperand(i: 0).getMBB();
1303 return false;
1304 } else if (LastInst.getOpcode() == PPC::BCC) {
1305 if (!LastInst.getOperand(i: 2).isMBB())
1306 return true;
1307 // Block ends with fall-through condbranch.
1308 TBB = LastInst.getOperand(i: 2).getMBB();
1309 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1310 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1311 return false;
1312 } else if (LastInst.getOpcode() == PPC::BC) {
1313 if (!LastInst.getOperand(i: 1).isMBB())
1314 return true;
1315 // Block ends with fall-through condbranch.
1316 TBB = LastInst.getOperand(i: 1).getMBB();
1317 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
1318 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1319 return false;
1320 } else if (LastInst.getOpcode() == PPC::BCn) {
1321 if (!LastInst.getOperand(i: 1).isMBB())
1322 return true;
1323 // Block ends with fall-through condbranch.
1324 TBB = LastInst.getOperand(i: 1).getMBB();
1325 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_UNSET));
1326 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1327 return false;
1328 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1329 LastInst.getOpcode() == PPC::BDNZ) {
1330 if (!LastInst.getOperand(i: 0).isMBB())
1331 return true;
1332 if (!EnableCTRLoopAnal)
1333 return true;
1334 TBB = LastInst.getOperand(i: 0).getMBB();
1335 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 1));
1336 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1337 isDef: true));
1338 return false;
1339 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1340 LastInst.getOpcode() == PPC::BDZ) {
1341 if (!LastInst.getOperand(i: 0).isMBB())
1342 return true;
1343 if (!EnableCTRLoopAnal)
1344 return true;
1345 TBB = LastInst.getOperand(i: 0).getMBB();
1346 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
1347 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1348 isDef: true));
1349 return false;
1350 }
1351
1352 // Otherwise, don't know what this is.
1353 return true;
1354 }
1355
1356 // Get the instruction before it if it's a terminator.
1357 MachineInstr &SecondLastInst = *I;
1358
1359 // If there are three terminators, we don't know what sort of block this is.
1360 if (I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
1361 return true;
1362
1363 // If the block ends with PPC::B and PPC:BCC, handle it.
1364 if (SecondLastInst.getOpcode() == PPC::BCC &&
1365 LastInst.getOpcode() == PPC::B) {
1366 if (!SecondLastInst.getOperand(i: 2).isMBB() ||
1367 !LastInst.getOperand(i: 0).isMBB())
1368 return true;
1369 TBB = SecondLastInst.getOperand(i: 2).getMBB();
1370 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1371 Cond.push_back(Elt: SecondLastInst.getOperand(i: 1));
1372 FBB = LastInst.getOperand(i: 0).getMBB();
1373 return false;
1374 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1375 LastInst.getOpcode() == PPC::B) {
1376 if (!SecondLastInst.getOperand(i: 1).isMBB() ||
1377 !LastInst.getOperand(i: 0).isMBB())
1378 return true;
1379 TBB = SecondLastInst.getOperand(i: 1).getMBB();
1380 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
1381 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1382 FBB = LastInst.getOperand(i: 0).getMBB();
1383 return false;
1384 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1385 LastInst.getOpcode() == PPC::B) {
1386 if (!SecondLastInst.getOperand(i: 1).isMBB() ||
1387 !LastInst.getOperand(i: 0).isMBB())
1388 return true;
1389 TBB = SecondLastInst.getOperand(i: 1).getMBB();
1390 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_UNSET));
1391 Cond.push_back(Elt: SecondLastInst.getOperand(i: 0));
1392 FBB = LastInst.getOperand(i: 0).getMBB();
1393 return false;
1394 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1395 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1396 LastInst.getOpcode() == PPC::B) {
1397 if (!SecondLastInst.getOperand(i: 0).isMBB() ||
1398 !LastInst.getOperand(i: 0).isMBB())
1399 return true;
1400 if (!EnableCTRLoopAnal)
1401 return true;
1402 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1403 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 1));
1404 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1405 isDef: true));
1406 FBB = LastInst.getOperand(i: 0).getMBB();
1407 return false;
1408 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1409 SecondLastInst.getOpcode() == PPC::BDZ) &&
1410 LastInst.getOpcode() == PPC::B) {
1411 if (!SecondLastInst.getOperand(i: 0).isMBB() ||
1412 !LastInst.getOperand(i: 0).isMBB())
1413 return true;
1414 if (!EnableCTRLoopAnal)
1415 return true;
1416 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1417 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
1418 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: isPPC64 ? PPC::CTR8 : PPC::CTR,
1419 isDef: true));
1420 FBB = LastInst.getOperand(i: 0).getMBB();
1421 return false;
1422 }
1423
1424 // If the block ends with two PPC:Bs, handle it. The second one is not
1425 // executed, so remove it.
1426 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1427 if (!SecondLastInst.getOperand(i: 0).isMBB())
1428 return true;
1429 TBB = SecondLastInst.getOperand(i: 0).getMBB();
1430 I = LastInst;
1431 if (AllowModify)
1432 I->eraseFromParent();
1433 return false;
1434 }
1435
1436 // Otherwise, can't handle this.
1437 return true;
1438}
1439
1440unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1441 int *BytesRemoved) const {
1442 assert(!BytesRemoved && "code size not handled");
1443
1444 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1445 if (I == MBB.end())
1446 return 0;
1447
1448 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1449 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1450 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1451 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1452 return 0;
1453
1454 // Remove the branch.
1455 I->eraseFromParent();
1456
1457 I = MBB.end();
1458
1459 if (I == MBB.begin()) return 1;
1460 --I;
1461 if (I->getOpcode() != PPC::BCC &&
1462 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1463 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1464 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1465 return 1;
1466
1467 // Remove the branch.
1468 I->eraseFromParent();
1469 return 2;
1470}
1471
1472unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1473 MachineBasicBlock *TBB,
1474 MachineBasicBlock *FBB,
1475 ArrayRef<MachineOperand> Cond,
1476 const DebugLoc &DL,
1477 int *BytesAdded) const {
1478 // Shouldn't be a fall through.
1479 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1480 assert((Cond.size() == 2 || Cond.size() == 0) &&
1481 "PPC branch conditions have two components!");
1482 assert(!BytesAdded && "code size not handled");
1483
1484 bool isPPC64 = Subtarget.isPPC64();
1485
1486 // One-way branch.
1487 if (!FBB) {
1488 if (Cond.empty()) // Unconditional branch
1489 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::B)).addMBB(MBB: TBB);
1490 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1491 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm() ?
1492 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1493 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(MBB: TBB);
1494 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1495 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BC)).add(MO: Cond[1]).addMBB(MBB: TBB);
1496 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1497 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCn)).add(MO: Cond[1]).addMBB(MBB: TBB);
1498 else // Conditional branch
1499 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCC))
1500 .addImm(Val: Cond[0].getImm())
1501 .add(MO: Cond[1])
1502 .addMBB(MBB: TBB);
1503 return 1;
1504 }
1505
1506 // Two-way Conditional Branch.
1507 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1508 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm() ?
1509 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1510 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(MBB: TBB);
1511 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1512 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BC)).add(MO: Cond[1]).addMBB(MBB: TBB);
1513 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1514 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCn)).add(MO: Cond[1]).addMBB(MBB: TBB);
1515 else
1516 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::BCC))
1517 .addImm(Val: Cond[0].getImm())
1518 .add(MO: Cond[1])
1519 .addMBB(MBB: TBB);
1520 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: PPC::B)).addMBB(MBB: FBB);
1521 return 2;
1522}
1523
1524// Select analysis.
1525bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1526 ArrayRef<MachineOperand> Cond,
1527 Register DstReg, Register TrueReg,
1528 Register FalseReg,
1529 SelectExpansion &Exp) const {
1530 if (!Subtarget.hasISEL())
1531 return false;
1532
1533 if (Cond.size() != 2)
1534 return false;
1535
1536 // If this is really a bdnz-like condition, then it cannot be turned into a
1537 // select.
1538 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1539 return false;
1540
1541 // If the conditional branch uses a physical register, then it cannot be
1542 // turned into a select.
1543 if (Cond[1].getReg().isPhysical())
1544 return false;
1545
1546 // Check register classes.
1547 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1548 const TargetRegisterClass *RC =
1549 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1550 if (!RC)
1551 return false;
1552
1553 // isel is for regular integer GPRs only.
1554 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1555 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1556 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1557 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1558 return false;
1559
1560 // FIXME: These numbers are for the A2, how well they work for other cores is
1561 // an open question. On the A2, the isel instruction has a 2-cycle latency
1562 // but single-cycle throughput. These numbers are used in combination with
1563 // the MispredictPenalty setting from the active SchedMachineModel.
1564 Exp.CondCycles = 1;
1565 Exp.TrueCycles = 1;
1566 Exp.FalseCycles = 1;
1567
1568 return true;
1569}
1570
1571void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1572 MachineBasicBlock::iterator MI,
1573 const DebugLoc &dl, Register DestReg,
1574 ArrayRef<MachineOperand> Cond, Register TrueReg,
1575 Register FalseReg) const {
1576 assert(Cond.size() == 2 &&
1577 "PPC branch conditions have two components!");
1578
1579 // Get the register classes.
1580 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1581 const TargetRegisterClass *RC =
1582 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1583 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1584
1585 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1586 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1587 assert((Is64Bit ||
1588 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1589 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1590 "isel is for regular integer GPRs only");
1591
1592 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1593 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1594
1595 unsigned SubIdx = 0;
1596 bool SwapOps = false;
1597 switch (SelectPred) {
1598 case PPC::PRED_EQ:
1599 case PPC::PRED_EQ_MINUS:
1600 case PPC::PRED_EQ_PLUS:
1601 SubIdx = PPC::sub_eq; SwapOps = false; break;
1602 case PPC::PRED_NE:
1603 case PPC::PRED_NE_MINUS:
1604 case PPC::PRED_NE_PLUS:
1605 SubIdx = PPC::sub_eq; SwapOps = true; break;
1606 case PPC::PRED_LT:
1607 case PPC::PRED_LT_MINUS:
1608 case PPC::PRED_LT_PLUS:
1609 SubIdx = PPC::sub_lt; SwapOps = false; break;
1610 case PPC::PRED_GE:
1611 case PPC::PRED_GE_MINUS:
1612 case PPC::PRED_GE_PLUS:
1613 SubIdx = PPC::sub_lt; SwapOps = true; break;
1614 case PPC::PRED_GT:
1615 case PPC::PRED_GT_MINUS:
1616 case PPC::PRED_GT_PLUS:
1617 SubIdx = PPC::sub_gt; SwapOps = false; break;
1618 case PPC::PRED_LE:
1619 case PPC::PRED_LE_MINUS:
1620 case PPC::PRED_LE_PLUS:
1621 SubIdx = PPC::sub_gt; SwapOps = true; break;
1622 case PPC::PRED_UN:
1623 case PPC::PRED_UN_MINUS:
1624 case PPC::PRED_UN_PLUS:
1625 SubIdx = PPC::sub_un; SwapOps = false; break;
1626 case PPC::PRED_NU:
1627 case PPC::PRED_NU_MINUS:
1628 case PPC::PRED_NU_PLUS:
1629 SubIdx = PPC::sub_un; SwapOps = true; break;
1630 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1631 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1632 }
1633
1634 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1635 SecondReg = SwapOps ? TrueReg : FalseReg;
1636
1637 // The first input register of isel cannot be r0. If it is a member
1638 // of a register class that can be r0, then copy it first (the
1639 // register allocator should eliminate the copy).
1640 if (MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::R0) ||
1641 MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::X0)) {
1642 const TargetRegisterClass *FirstRC =
1643 MRI.getRegClass(Reg: FirstReg)->contains(Reg: PPC::X0) ?
1644 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1645 Register OldFirstReg = FirstReg;
1646 FirstReg = MRI.createVirtualRegister(RegClass: FirstRC);
1647 BuildMI(BB&: MBB, I: MI, MIMD: dl, MCID: get(Opcode: TargetOpcode::COPY), DestReg: FirstReg)
1648 .addReg(RegNo: OldFirstReg);
1649 }
1650
1651 BuildMI(BB&: MBB, I: MI, MIMD: dl, MCID: get(Opcode: OpCode), DestReg)
1652 .addReg(RegNo: FirstReg)
1653 .addReg(RegNo: SecondReg)
1654 .addReg(RegNo: Cond[1].getReg(), Flags: {}, SubReg: SubIdx);
1655}
1656
1657static unsigned getCRBitValue(unsigned CRBit) {
1658 unsigned Ret = 4;
1659 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1660 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1661 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1662 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1663 Ret = 3;
1664 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1665 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1666 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1667 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1668 Ret = 2;
1669 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1670 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1671 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1672 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1673 Ret = 1;
1674 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1675 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1676 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1677 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1678 Ret = 0;
1679
1680 assert(Ret != 4 && "Invalid CR bit register");
1681 return Ret;
1682}
1683
1684void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1685 MachineBasicBlock::iterator I,
1686 const DebugLoc &DL, Register DestReg,
1687 Register SrcReg, bool KillSrc,
1688 bool RenamableDest, bool RenamableSrc) const {
1689 // We can end up with self copies and similar things as a result of VSX copy
1690 // legalization. Promote them here.
1691 if (PPC::F8RCRegClass.contains(Reg: DestReg) &&
1692 PPC::VSRCRegClass.contains(Reg: SrcReg)) {
1693 MCRegister SuperReg =
1694 RI.getMatchingSuperReg(Reg: DestReg, SubIdx: PPC::sub_64, RC: &PPC::VSRCRegClass);
1695
1696 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1697 llvm_unreachable("nop VSX copy");
1698
1699 DestReg = SuperReg;
1700 } else if (PPC::F8RCRegClass.contains(Reg: SrcReg) &&
1701 PPC::VSRCRegClass.contains(Reg: DestReg)) {
1702 MCRegister SuperReg =
1703 RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: PPC::sub_64, RC: &PPC::VSRCRegClass);
1704
1705 if (VSXSelfCopyCrash && DestReg == SuperReg)
1706 llvm_unreachable("nop VSX copy");
1707
1708 SrcReg = SuperReg;
1709 }
1710
1711 // Different class register copy
1712 if (PPC::CRBITRCRegClass.contains(Reg: SrcReg) &&
1713 PPC::GPRCRegClass.contains(Reg: DestReg)) {
1714 MCRegister CRReg = getCRFromCRBit(SrcReg);
1715 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MFOCRF), DestReg).addReg(RegNo: CRReg);
1716 getKillRegState(B: KillSrc);
1717 // Rotate the CR bit in the CR fields to be the least significant bit and
1718 // then mask with 0x1 (MB = ME = 31).
1719 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::RLWINM), DestReg)
1720 .addReg(RegNo: DestReg, Flags: RegState::Kill)
1721 .addImm(Val: RI.getEncodingValue(Reg: CRReg) * 4 + (4 - getCRBitValue(CRBit: SrcReg)))
1722 .addImm(Val: 31)
1723 .addImm(Val: 31);
1724 return;
1725 } else if (PPC::CRRCRegClass.contains(Reg: SrcReg) &&
1726 (PPC::G8RCRegClass.contains(Reg: DestReg) ||
1727 PPC::GPRCRegClass.contains(Reg: DestReg))) {
1728 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: DestReg);
1729 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1730 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1731 unsigned CRNum = RI.getEncodingValue(Reg: SrcReg);
1732 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: MvCode), DestReg).addReg(RegNo: SrcReg);
1733 getKillRegState(B: KillSrc);
1734 if (CRNum == 7)
1735 return;
1736 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1737 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ShCode), DestReg)
1738 .addReg(RegNo: DestReg, Flags: RegState::Kill)
1739 .addImm(Val: CRNum * 4 + 4)
1740 .addImm(Val: 28)
1741 .addImm(Val: 31);
1742 return;
1743 } else if (PPC::G8RCRegClass.contains(Reg: SrcReg) &&
1744 PPC::VSFRCRegClass.contains(Reg: DestReg)) {
1745 assert(Subtarget.hasDirectMove() &&
1746 "Subtarget doesn't support directmove, don't know how to copy.");
1747 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MTVSRD), DestReg).addReg(RegNo: SrcReg);
1748 NumGPRtoVSRSpill++;
1749 getKillRegState(B: KillSrc);
1750 return;
1751 } else if (PPC::VSFRCRegClass.contains(Reg: SrcReg) &&
1752 PPC::G8RCRegClass.contains(Reg: DestReg)) {
1753 assert(Subtarget.hasDirectMove() &&
1754 "Subtarget doesn't support directmove, don't know how to copy.");
1755 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::MFVSRD), DestReg).addReg(RegNo: SrcReg);
1756 getKillRegState(B: KillSrc);
1757 return;
1758 } else if (PPC::SPERCRegClass.contains(Reg: SrcReg) &&
1759 PPC::GPRCRegClass.contains(Reg: DestReg)) {
1760 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::EFSCFD), DestReg).addReg(RegNo: SrcReg);
1761 getKillRegState(B: KillSrc);
1762 return;
1763 } else if (PPC::GPRCRegClass.contains(Reg: SrcReg) &&
1764 PPC::SPERCRegClass.contains(Reg: DestReg)) {
1765 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::EFDCFS), DestReg).addReg(RegNo: SrcReg);
1766 getKillRegState(B: KillSrc);
1767 return;
1768 } else if ((PPC::G8RCRegClass.contains(Reg: DestReg) ||
1769 PPC::GPRCRegClass.contains(Reg: DestReg)) &&
1770 SrcReg == PPC::CARRY) {
1771 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: DestReg);
1772 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Is64Bit ? PPC::MFSPR8 : PPC::MFSPR), DestReg)
1773 .addImm(Val: 1)
1774 .addReg(RegNo: PPC::CARRY, Flags: RegState::Implicit);
1775 return;
1776 } else if ((PPC::G8RCRegClass.contains(Reg: SrcReg) ||
1777 PPC::GPRCRegClass.contains(Reg: SrcReg)) &&
1778 DestReg == PPC::CARRY) {
1779 bool Is64Bit = PPC::G8RCRegClass.contains(Reg: SrcReg);
1780 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Is64Bit ? PPC::MTSPR8 : PPC::MTSPR))
1781 .addImm(Val: 1)
1782 .addReg(RegNo: SrcReg)
1783 .addReg(RegNo: PPC::CARRY, Flags: RegState::ImplicitDefine);
1784 return;
1785 }
1786
1787 unsigned Opc;
1788 if (PPC::GPRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1789 Opc = PPC::OR;
1790 else if (PPC::G8RCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1791 Opc = PPC::OR8;
1792 else if (PPC::F4RCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1793 Opc = PPC::FMR;
1794 else if (PPC::CRRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1795 Opc = PPC::MCRF;
1796 else if (PPC::VRRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1797 Opc = PPC::VOR;
1798 else if (PPC::VSRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1799 // There are two different ways this can be done:
1800 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1801 // issue in VSU pipeline 0.
1802 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1803 // can go to either pipeline.
1804 // We'll always use xxlor here, because in practically all cases where
1805 // copies are generated, they are close enough to some use that the
1806 // lower-latency form is preferable.
1807 Opc = PPC::XXLOR;
1808 else if (PPC::VSFRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg) ||
1809 PPC::VSSRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1810 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1811 else if (Subtarget.pairedVectorMemops() &&
1812 PPC::VSRpRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
1813 if (SrcReg > PPC::VSRp15)
1814 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1815 else
1816 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1817 if (DestReg > PPC::VSRp15)
1818 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1819 else
1820 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1821 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg).
1822 addReg(RegNo: SrcReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1823 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: DestReg + 1).
1824 addReg(RegNo: SrcReg + 1).addReg(RegNo: SrcReg + 1, Flags: getKillRegState(B: KillSrc));
1825 return;
1826 }
1827 else if (PPC::CRBITRCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1828 Opc = PPC::CROR;
1829 else if (PPC::SPERCRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
1830 Opc = PPC::EVOR;
1831 else if ((PPC::ACCRCRegClass.contains(Reg: DestReg) ||
1832 PPC::UACCRCRegClass.contains(Reg: DestReg)) &&
1833 (PPC::ACCRCRegClass.contains(Reg: SrcReg) ||
1834 PPC::UACCRCRegClass.contains(Reg: SrcReg))) {
1835 // If primed, de-prime the source register, copy the individual registers
1836 // and prime the destination if needed. The vector subregisters are
1837 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1838 // source is primed, we need to re-prime it after the copy as well.
1839 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1840 bool DestPrimed = PPC::ACCRCRegClass.contains(Reg: DestReg);
1841 bool SrcPrimed = PPC::ACCRCRegClass.contains(Reg: SrcReg);
1842 MCRegister VSLSrcReg =
1843 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1844 MCRegister VSLDestReg =
1845 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1846 if (SrcPrimed)
1847 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMFACC), DestReg: SrcReg).addReg(RegNo: SrcReg);
1848 for (unsigned Idx = 0; Idx < 4; Idx++)
1849 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: VSLDestReg + Idx)
1850 .addReg(RegNo: VSLSrcReg + Idx)
1851 .addReg(RegNo: VSLSrcReg + Idx, Flags: getKillRegState(B: KillSrc));
1852 if (DestPrimed)
1853 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMTACC), DestReg).addReg(RegNo: DestReg);
1854 if (SrcPrimed && !KillSrc)
1855 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::XXMTACC), DestReg: SrcReg).addReg(RegNo: SrcReg);
1856 return;
1857 } else if (PPC::G8pRCRegClass.contains(Reg: DestReg) &&
1858 PPC::G8pRCRegClass.contains(Reg: SrcReg)) {
1859 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1860 unsigned DestRegIdx = DestReg - PPC::G8p0;
1861 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1862 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1863 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1864 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1865 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1866 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DestRegSub0)
1867 .addReg(RegNo: SrcRegSub0)
1868 .addReg(RegNo: SrcRegSub0, Flags: getKillRegState(B: KillSrc));
1869 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DestRegSub1)
1870 .addReg(RegNo: SrcRegSub1)
1871 .addReg(RegNo: SrcRegSub1, Flags: getKillRegState(B: KillSrc));
1872 return;
1873 } else if ((PPC::WACCRCRegClass.contains(Reg: DestReg) ||
1874 PPC::WACC_HIRCRegClass.contains(Reg: DestReg)) &&
1875 (PPC::WACCRCRegClass.contains(Reg: SrcReg) ||
1876 PPC::WACC_HIRCRegClass.contains(Reg: SrcReg))) {
1877
1878 Opc = PPC::WACCRCRegClass.contains(Reg: SrcReg) ? PPC::DMXXEXTFDMR512
1879 : PPC::DMXXEXTFDMR512_HI;
1880
1881 RegScavenger RS;
1882 RS.enterBasicBlockEnd(MBB);
1883 RS.backward(I: std::next(x: I));
1884
1885 Register TmpReg1 = RS.scavengeRegisterBackwards(RC: PPC::VSRpRCRegClass, To: I,
1886 /* RestoreAfter */ false, SPAdj: 0,
1887 /* AllowSpill */ false);
1888
1889 RS.setRegUsed(Reg: TmpReg1);
1890 Register TmpReg2 = RS.scavengeRegisterBackwards(RC: PPC::VSRpRCRegClass, To: I,
1891 /* RestoreAfter */ false, SPAdj: 0,
1892 /* AllowSpill */ false);
1893
1894 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc))
1895 .addReg(RegNo: TmpReg1, Flags: RegState::Define)
1896 .addReg(RegNo: TmpReg2, Flags: RegState::Define)
1897 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1898
1899 Opc = PPC::WACCRCRegClass.contains(Reg: DestReg) ? PPC::DMXXINSTDMR512
1900 : PPC::DMXXINSTDMR512_HI;
1901
1902 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
1903 .addReg(RegNo: TmpReg1, Flags: RegState::Kill)
1904 .addReg(RegNo: TmpReg2, Flags: RegState::Kill);
1905
1906 return;
1907 } else if (PPC::DMRRCRegClass.contains(Reg: DestReg) &&
1908 PPC::DMRRCRegClass.contains(Reg: SrcReg)) {
1909
1910 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: PPC::DMMR), DestReg)
1911 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1912
1913 return;
1914
1915 } else
1916 llvm_unreachable("Impossible reg-to-reg copy");
1917
1918 const MCInstrDesc &MCID = get(Opcode: Opc);
1919 if (MCID.getNumOperands() == 3)
1920 BuildMI(BB&: MBB, I, MIMD: DL, MCID, DestReg)
1921 .addReg(RegNo: SrcReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1922 else
1923 BuildMI(BB&: MBB, I, MIMD: DL, MCID, DestReg).addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1924}
1925
1926unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1927 int OpcodeIndex = 0;
1928
1929 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1930 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1931 OpcodeIndex = SOK_Int4Spill;
1932 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1933 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1934 OpcodeIndex = SOK_Int8Spill;
1935 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1936 OpcodeIndex = SOK_Float8Spill;
1937 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1938 OpcodeIndex = SOK_Float4Spill;
1939 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1940 OpcodeIndex = SOK_SPESpill;
1941 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1942 OpcodeIndex = SOK_CRSpill;
1943 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1944 OpcodeIndex = SOK_CRBitSpill;
1945 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1946 OpcodeIndex = SOK_VRVectorSpill;
1947 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1948 OpcodeIndex = SOK_VSXVectorSpill;
1949 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1950 OpcodeIndex = SOK_VectorFloat8Spill;
1951 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1952 OpcodeIndex = SOK_VectorFloat4Spill;
1953 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1954 OpcodeIndex = SOK_SpillToVSR;
1955 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1956 assert(Subtarget.pairedVectorMemops() &&
1957 "Register unexpected when paired memops are disabled.");
1958 OpcodeIndex = SOK_AccumulatorSpill;
1959 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1960 assert(Subtarget.pairedVectorMemops() &&
1961 "Register unexpected when paired memops are disabled.");
1962 OpcodeIndex = SOK_UAccumulatorSpill;
1963 } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1964 assert(Subtarget.pairedVectorMemops() &&
1965 "Register unexpected when paired memops are disabled.");
1966 OpcodeIndex = SOK_WAccumulatorSpill;
1967 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1968 assert(Subtarget.pairedVectorMemops() &&
1969 "Register unexpected when paired memops are disabled.");
1970 OpcodeIndex = SOK_PairedVecSpill;
1971 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1972 OpcodeIndex = SOK_PairedG8Spill;
1973 } else if (PPC::DMRROWRCRegClass.hasSubClassEq(RC)) {
1974 llvm_unreachable("TODO: Implement spill DMRROW regclass!");
1975 } else if (PPC::DMRROWpRCRegClass.hasSubClassEq(RC)) {
1976 llvm_unreachable("TODO: Implement spill DMRROWp regclass!");
1977 } else if (PPC::DMRpRCRegClass.hasSubClassEq(RC)) {
1978 OpcodeIndex = SOK_DMRpSpill;
1979 } else if (PPC::DMRRCRegClass.hasSubClassEq(RC)) {
1980 OpcodeIndex = SOK_DMRSpill;
1981 } else {
1982 llvm_unreachable("Unknown regclass!");
1983 }
1984 return OpcodeIndex;
1985}
1986
1987unsigned
1988PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1989 ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1990 return OpcodesForSpill[getSpillIndex(RC)];
1991}
1992
1993unsigned
1994PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1995 ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1996 return OpcodesForSpill[getSpillIndex(RC)];
1997}
1998
1999void PPCInstrInfo::StoreRegToStackSlot(
2000 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
2001 const TargetRegisterClass *RC,
2002 SmallVectorImpl<MachineInstr *> &NewMIs) const {
2003 unsigned Opcode = getStoreOpcodeForSpill(RC);
2004 DebugLoc DL;
2005
2006 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2007 FuncInfo->setHasSpills();
2008
2009 NewMIs.push_back(Elt: addFrameReference(
2010 MIB: BuildMI(MF, MIMD: DL, MCID: get(Opcode)).addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)),
2011 FI: FrameIdx));
2012
2013 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
2014 PPC::CRBITRCRegClass.hasSubClassEq(RC))
2015 FuncInfo->setSpillsCR();
2016
2017 if (isXFormMemOp(Opcode))
2018 FuncInfo->setHasNonRISpills();
2019}
2020
2021void PPCInstrInfo::storeRegToStackSlotNoUpd(
2022 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
2023 bool isKill, int FrameIdx, const TargetRegisterClass *RC) const {
2024 MachineFunction &MF = *MBB.getParent();
2025 SmallVector<MachineInstr *, 4> NewMIs;
2026
2027 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
2028
2029 for (MachineInstr *NewMI : NewMIs)
2030 MBB.insert(I: MI, MI: NewMI);
2031
2032 const MachineFrameInfo &MFI = MF.getFrameInfo();
2033 MachineMemOperand *MMO = MF.getMachineMemOperand(
2034 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
2035 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
2036 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
2037 NewMIs.back()->addMemOperand(MF, MO: MMO);
2038}
2039
2040void PPCInstrInfo::storeRegToStackSlot(
2041 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
2042 bool isKill, int FrameIdx, const TargetRegisterClass *RC, Register VReg,
2043 MachineInstr::MIFlag Flags) const {
2044 // We need to avoid a situation in which the value from a VRRC register is
2045 // spilled using an Altivec instruction and reloaded into a VSRC register
2046 // using a VSX instruction. The issue with this is that the VSX
2047 // load/store instructions swap the doublewords in the vector and the Altivec
2048 // ones don't. The register classes on the spill/reload may be different if
2049 // the register is defined using an Altivec instruction and is then used by a
2050 // VSX instruction.
2051 RC = updatedRC(RC);
2052 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC);
2053}
2054
2055void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
2056 unsigned DestReg, int FrameIdx,
2057 const TargetRegisterClass *RC,
2058 SmallVectorImpl<MachineInstr *> &NewMIs)
2059 const {
2060 unsigned Opcode = getLoadOpcodeForSpill(RC);
2061 NewMIs.push_back(Elt: addFrameReference(MIB: BuildMI(MF, MIMD: DL, MCID: get(Opcode), DestReg),
2062 FI: FrameIdx));
2063}
2064
2065void PPCInstrInfo::loadRegFromStackSlotNoUpd(
2066 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
2067 int FrameIdx, const TargetRegisterClass *RC) const {
2068 MachineFunction &MF = *MBB.getParent();
2069 SmallVector<MachineInstr*, 4> NewMIs;
2070 DebugLoc DL;
2071 if (MI != MBB.end()) DL = MI->getDebugLoc();
2072
2073 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2074
2075 for (MachineInstr *NewMI : NewMIs)
2076 MBB.insert(I: MI, MI: NewMI);
2077
2078 const MachineFrameInfo &MFI = MF.getFrameInfo();
2079 MachineMemOperand *MMO = MF.getMachineMemOperand(
2080 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
2081 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
2082 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
2083 NewMIs.back()->addMemOperand(MF, MO: MMO);
2084}
2085
2086void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
2087 MachineBasicBlock::iterator MI,
2088 Register DestReg, int FrameIdx,
2089 const TargetRegisterClass *RC,
2090 Register VReg, unsigned SubReg,
2091 MachineInstr::MIFlag Flags) const {
2092 // We need to avoid a situation in which the value from a VRRC register is
2093 // spilled using an Altivec instruction and reloaded into a VSRC register
2094 // using a VSX instruction. The issue with this is that the VSX
2095 // load/store instructions swap the doublewords in the vector and the Altivec
2096 // ones don't. The register classes on the spill/reload may be different if
2097 // the register is defined using an Altivec instruction and is then used by a
2098 // VSX instruction.
2099 RC = updatedRC(RC);
2100
2101 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC);
2102}
2103
2104bool PPCInstrInfo::
2105reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
2106 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2107 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2108 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2109 else
2110 // Leave the CR# the same, but invert the condition.
2111 Cond[0].setImm(PPC::InvertPredicate(Opcode: (PPC::Predicate)Cond[0].getImm()));
2112 return false;
2113}
2114
2115// For some instructions, it is legal to fold ZERO into the RA register field.
2116// This function performs that fold by replacing the operand with PPC::ZERO,
2117// it does not consider whether the load immediate zero is no longer in use.
2118bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2119 Register Reg) const {
2120 // A zero immediate should always be loaded with a single li.
2121 unsigned DefOpc = DefMI.getOpcode();
2122 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2123 return false;
2124 if (!DefMI.getOperand(i: 1).isImm())
2125 return false;
2126 if (DefMI.getOperand(i: 1).getImm() != 0)
2127 return false;
2128
2129 // Note that we cannot here invert the arguments of an isel in order to fold
2130 // a ZERO into what is presented as the second argument. All we have here
2131 // is the condition bit, and that might come from a CR-logical bit operation.
2132
2133 const MCInstrDesc &UseMCID = UseMI.getDesc();
2134
2135 // Only fold into real machine instructions.
2136 if (UseMCID.isPseudo())
2137 return false;
2138
2139 // We need to find which of the User's operands is to be folded, that will be
2140 // the operand that matches the given register ID.
2141 unsigned UseIdx;
2142 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2143 if (UseMI.getOperand(i: UseIdx).isReg() &&
2144 UseMI.getOperand(i: UseIdx).getReg() == Reg)
2145 break;
2146
2147 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2148 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2149
2150 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2151 // register (which might also be specified as a pointer class kind).
2152
2153 const MCOperandInfo &UseInfo = UseMCID.operands()[UseIdx];
2154 int16_t RegClass = getOpRegClassID(OpInfo: UseInfo);
2155 if (UseInfo.RegClass != PPC::GPRC_NOR0RegClassID &&
2156 UseInfo.RegClass != PPC::G8RC_NOX0RegClassID)
2157 return false;
2158
2159 // Make sure this is not tied to an output register (or otherwise
2160 // constrained). This is true for ST?UX registers, for example, which
2161 // are tied to their output registers.
2162 if (UseInfo.Constraints != 0)
2163 return false;
2164
2165 MCRegister ZeroReg =
2166 RegClass == PPC::G8RC_NOX0RegClassID ? PPC::ZERO8 : PPC::ZERO;
2167
2168 LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2169 LLVM_DEBUG(UseMI.dump());
2170 UseMI.getOperand(i: UseIdx).setReg(ZeroReg);
2171 LLVM_DEBUG(dbgs() << "Into: ");
2172 LLVM_DEBUG(UseMI.dump());
2173 return true;
2174}
2175
2176// Folds zero into instructions which have a load immediate zero as an operand
2177// but also recognize zero as immediate zero. If the definition of the load
2178// has no more users it is deleted.
2179bool PPCInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2180 Register Reg, MachineRegisterInfo *MRI) const {
2181 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2182 if (MRI->use_nodbg_empty(RegNo: Reg))
2183 DefMI.eraseFromParent();
2184 return Changed;
2185}
2186
2187static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
2188 for (MachineInstr &MI : MBB)
2189 if (MI.definesRegister(Reg: PPC::CTR, /*TRI=*/nullptr) ||
2190 MI.definesRegister(Reg: PPC::CTR8, /*TRI=*/nullptr))
2191 return true;
2192 return false;
2193}
2194
2195// We should make sure that, if we're going to predicate both sides of a
2196// condition (a diamond), that both sides don't define the counter register. We
2197// can predicate counter-decrement-based branches, but while that predicates
2198// the branching, it does not predicate the counter decrement. If we tried to
2199// merge the triangle into one predicated block, we'd decrement the counter
2200// twice.
2201bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
2202 unsigned NumT, unsigned ExtraT,
2203 MachineBasicBlock &FMBB,
2204 unsigned NumF, unsigned ExtraF,
2205 BranchProbability Probability) const {
2206 return !(MBBDefinesCTR(MBB&: TMBB) && MBBDefinesCTR(MBB&: FMBB));
2207}
2208
2209
2210bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
2211 // The predicated branches are identified by their type, not really by the
2212 // explicit presence of a predicate. Furthermore, some of them can be
2213 // predicated more than once. Because if conversion won't try to predicate
2214 // any instruction which already claims to be predicated (by returning true
2215 // here), always return false. In doing so, we let isPredicable() be the
2216 // final word on whether not the instruction can be (further) predicated.
2217
2218 return false;
2219}
2220
2221bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2222 const MachineBasicBlock *MBB,
2223 const MachineFunction &MF) const {
2224 switch (MI.getOpcode()) {
2225 default:
2226 break;
2227 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2228 // across them, since some FP operations may change content of FPSCR.
2229 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2230 case PPC::MFFS:
2231 case PPC::MTFSF:
2232 case PPC::FENCE:
2233 return true;
2234 }
2235 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
2236}
2237
2238bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
2239 ArrayRef<MachineOperand> Pred) const {
2240 unsigned OpC = MI.getOpcode();
2241 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2242 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2243 bool isPPC64 = Subtarget.isPPC64();
2244 MI.setDesc(get(Opcode: Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2245 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2246 // Need add Def and Use for CTR implicit operand.
2247 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2248 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::Implicit)
2249 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::ImplicitDefine);
2250 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2251 MI.setDesc(get(Opcode: PPC::BCLR));
2252 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2253 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2254 MI.setDesc(get(Opcode: PPC::BCLRn));
2255 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2256 } else {
2257 MI.setDesc(get(Opcode: PPC::BCCLR));
2258 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2259 .addImm(Val: Pred[0].getImm())
2260 .add(MO: Pred[1]);
2261 }
2262
2263 return true;
2264 } else if (OpC == PPC::B) {
2265 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2266 bool isPPC64 = Subtarget.isPPC64();
2267 MI.setDesc(get(Opcode: Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2268 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2269 // Need add Def and Use for CTR implicit operand.
2270 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2271 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::Implicit)
2272 .addReg(RegNo: Pred[1].getReg(), Flags: RegState::ImplicitDefine);
2273 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2274 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2275 MI.removeOperand(OpNo: 0);
2276
2277 MI.setDesc(get(Opcode: PPC::BC));
2278 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2279 .add(MO: Pred[1])
2280 .addMBB(MBB);
2281 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2282 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2283 MI.removeOperand(OpNo: 0);
2284
2285 MI.setDesc(get(Opcode: PPC::BCn));
2286 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2287 .add(MO: Pred[1])
2288 .addMBB(MBB);
2289 } else {
2290 MachineBasicBlock *MBB = MI.getOperand(i: 0).getMBB();
2291 MI.removeOperand(OpNo: 0);
2292
2293 MI.setDesc(get(Opcode: PPC::BCC));
2294 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2295 .addImm(Val: Pred[0].getImm())
2296 .add(MO: Pred[1])
2297 .addMBB(MBB);
2298 }
2299
2300 return true;
2301 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2302 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2303 OpC == PPC::BCTRL8_RM) {
2304 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2305 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2306
2307 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2308 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2309 bool isPPC64 = Subtarget.isPPC64();
2310
2311 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2312 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2313 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2314 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2315 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2316 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2317 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2318 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(MO: Pred[1]);
2319 } else {
2320 MI.setDesc(get(Opcode: isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2321 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2322 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2323 .addImm(Val: Pred[0].getImm())
2324 .add(MO: Pred[1]);
2325 }
2326
2327 // Need add Def and Use for LR implicit operand.
2328 if (setLR)
2329 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2330 .addReg(RegNo: isPPC64 ? PPC::LR8 : PPC::LR, Flags: RegState::Implicit)
2331 .addReg(RegNo: isPPC64 ? PPC::LR8 : PPC::LR, Flags: RegState::ImplicitDefine);
2332 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2333 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2334 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
2335
2336 return true;
2337 }
2338
2339 return false;
2340}
2341
2342bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
2343 ArrayRef<MachineOperand> Pred2) const {
2344 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2345 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2346
2347 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2348 return false;
2349 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2350 return false;
2351
2352 // P1 can only subsume P2 if they test the same condition register.
2353 if (Pred1[1].getReg() != Pred2[1].getReg())
2354 return false;
2355
2356 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2357 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2358
2359 if (P1 == P2)
2360 return true;
2361
2362 // Does P1 subsume P2, e.g. GE subsumes GT.
2363 if (P1 == PPC::PRED_LE &&
2364 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2365 return true;
2366 if (P1 == PPC::PRED_GE &&
2367 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2368 return true;
2369
2370 return false;
2371}
2372
2373bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
2374 std::vector<MachineOperand> &Pred,
2375 bool SkipDead) const {
2376 // Note: At the present time, the contents of Pred from this function is
2377 // unused by IfConversion. This implementation follows ARM by pushing the
2378 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2379 // predicate, instructions defining CTR or CTR8 are also included as
2380 // predicate-defining instructions.
2381
2382 const TargetRegisterClass *RCs[] =
2383 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2384 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2385
2386 bool Found = false;
2387 for (const MachineOperand &MO : MI.operands()) {
2388 for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2389 const TargetRegisterClass *RC = RCs[c];
2390 if (MO.isReg()) {
2391 if (MO.isDef() && RC->contains(Reg: MO.getReg())) {
2392 Pred.push_back(x: MO);
2393 Found = true;
2394 }
2395 } else if (MO.isRegMask()) {
2396 for (MCPhysReg R : *RC)
2397 if (MO.clobbersPhysReg(PhysReg: R)) {
2398 Pred.push_back(x: MO);
2399 Found = true;
2400 }
2401 }
2402 }
2403 }
2404
2405 return Found;
2406}
2407
2408bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2409 Register &SrcReg2, int64_t &Mask,
2410 int64_t &Value) const {
2411 unsigned Opc = MI.getOpcode();
2412
2413 switch (Opc) {
2414 default: return false;
2415 case PPC::CMPWI:
2416 case PPC::CMPLWI:
2417 case PPC::CMPDI:
2418 case PPC::CMPLDI:
2419 SrcReg = MI.getOperand(i: 1).getReg();
2420 SrcReg2 = 0;
2421 Value = MI.getOperand(i: 2).getImm();
2422 Mask = 0xFFFF;
2423 return true;
2424 case PPC::CMPW:
2425 case PPC::CMPLW:
2426 case PPC::CMPD:
2427 case PPC::CMPLD:
2428 case PPC::FCMPUS:
2429 case PPC::FCMPUD:
2430 SrcReg = MI.getOperand(i: 1).getReg();
2431 SrcReg2 = MI.getOperand(i: 2).getReg();
2432 Value = 0;
2433 Mask = 0;
2434 return true;
2435 }
2436}
2437
2438bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
2439 Register SrcReg2, int64_t Mask,
2440 int64_t Value,
2441 const MachineRegisterInfo *MRI) const {
2442 if (!EnableCmpOpt)
2443 return false;
2444
2445 int OpC = CmpInstr.getOpcode();
2446 Register CRReg = CmpInstr.getOperand(i: 0).getReg();
2447
2448 // FP record forms set CR1 based on the exception status bits, not a
2449 // comparison with zero.
2450 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2451 return false;
2452
2453 // The record forms set the condition register based on a signed comparison
2454 // with zero (so says the ISA manual). This is not as straightforward as it
2455 // seems, however, because this is always a 64-bit comparison on PPC64, even
2456 // for instructions that are 32-bit in nature (like slw for example).
2457 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2458 // for equality checks (as those don't depend on the sign). On PPC64,
2459 // we are restricted to equality for unsigned 64-bit comparisons and for
2460 // signed 32-bit comparisons the applicability is more restricted.
2461 bool isPPC64 = Subtarget.isPPC64();
2462 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2463 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2464 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2465
2466 // Look through copies unless that gets us to a physical register.
2467 Register ActualSrc = RI.lookThruCopyLike(SrcReg, MRI);
2468 if (ActualSrc.isVirtual())
2469 SrcReg = ActualSrc;
2470
2471 // Get the unique definition of SrcReg.
2472 MachineInstr *MI = MRI->getUniqueVRegDef(Reg: SrcReg);
2473 if (!MI) return false;
2474
2475 bool equalityOnly = false;
2476 bool noSub = false;
2477 if (isPPC64) {
2478 if (is32BitSignedCompare) {
2479 // We can perform this optimization only if SrcReg is sign-extending.
2480 if (isSignExtended(Reg: SrcReg, MRI))
2481 noSub = true;
2482 else
2483 return false;
2484 } else if (is32BitUnsignedCompare) {
2485 // We can perform this optimization, equality only, if SrcReg is
2486 // zero-extending.
2487 if (isZeroExtended(Reg: SrcReg, MRI)) {
2488 noSub = true;
2489 equalityOnly = true;
2490 } else
2491 return false;
2492 } else
2493 equalityOnly = is64BitUnsignedCompare;
2494 } else
2495 equalityOnly = is32BitUnsignedCompare;
2496
2497 if (equalityOnly) {
2498 // We need to check the uses of the condition register in order to reject
2499 // non-equality comparisons.
2500 for (MachineRegisterInfo::use_instr_iterator
2501 I = MRI->use_instr_begin(RegNo: CRReg), IE = MRI->use_instr_end();
2502 I != IE; ++I) {
2503 MachineInstr *UseMI = &*I;
2504 if (UseMI->getOpcode() == PPC::BCC) {
2505 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(i: 0).getImm();
2506 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2507 // We ignore hint bits when checking for non-equality comparisons.
2508 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2509 return false;
2510 } else if (UseMI->getOpcode() == PPC::ISEL ||
2511 UseMI->getOpcode() == PPC::ISEL8) {
2512 unsigned SubIdx = UseMI->getOperand(i: 3).getSubReg();
2513 if (SubIdx != PPC::sub_eq)
2514 return false;
2515 } else
2516 return false;
2517 }
2518 }
2519
2520 MachineBasicBlock::iterator I = CmpInstr;
2521
2522 // Scan forward to find the first use of the compare.
2523 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2524 ++I) {
2525 bool FoundUse = false;
2526 for (MachineRegisterInfo::use_instr_iterator
2527 J = MRI->use_instr_begin(RegNo: CRReg), JE = MRI->use_instr_end();
2528 J != JE; ++J)
2529 if (&*J == &*I) {
2530 FoundUse = true;
2531 break;
2532 }
2533
2534 if (FoundUse)
2535 break;
2536 }
2537
2538 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
2539 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
2540
2541 // There are two possible candidates which can be changed to set CR[01].
2542 // One is MI, the other is a SUB instruction.
2543 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2544 MachineInstr *Sub = nullptr;
2545 if (SrcReg2 != 0)
2546 // MI is not a candidate for CMPrr.
2547 MI = nullptr;
2548 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2549 // same BB as the comparison. This is to allow the check below to avoid calls
2550 // (and other explicit clobbers); instead we should really check for these
2551 // more explicitly (in at least a few predecessors).
2552 else if (MI->getParent() != CmpInstr.getParent())
2553 return false;
2554 else if (Value != 0) {
2555 // The record-form instructions set CR bit based on signed comparison
2556 // against 0. We try to convert a compare against 1 or -1 into a compare
2557 // against 0 to exploit record-form instructions. For example, we change
2558 // the condition "greater than -1" into "greater than or equal to 0"
2559 // and "less than 1" into "less than or equal to 0".
2560
2561 // Since we optimize comparison based on a specific branch condition,
2562 // we don't optimize if condition code is used by more than once.
2563 if (equalityOnly || !MRI->hasOneUse(RegNo: CRReg))
2564 return false;
2565
2566 MachineInstr *UseMI = &*MRI->use_instr_begin(RegNo: CRReg);
2567 if (UseMI->getOpcode() != PPC::BCC)
2568 return false;
2569
2570 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(i: 0).getImm();
2571 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2572 unsigned PredHint = PPC::getPredicateHint(Opcode: Pred);
2573 int16_t Immed = (int16_t)Value;
2574
2575 // When modifying the condition in the predicate, we propagate hint bits
2576 // from the original predicate to the new one.
2577 if (Immed == -1 && PredCond == PPC::PRED_GT)
2578 // We convert "greater than -1" into "greater than or equal to 0",
2579 // since we are assuming signed comparison by !equalityOnly
2580 Pred = PPC::getPredicate(Condition: PPC::PRED_GE, Hint: PredHint);
2581 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2582 // We convert "less than or equal to -1" into "less than 0".
2583 Pred = PPC::getPredicate(Condition: PPC::PRED_LT, Hint: PredHint);
2584 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2585 // We convert "less than 1" into "less than or equal to 0".
2586 Pred = PPC::getPredicate(Condition: PPC::PRED_LE, Hint: PredHint);
2587 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2588 // We convert "greater than or equal to 1" into "greater than 0".
2589 Pred = PPC::getPredicate(Condition: PPC::PRED_GT, Hint: PredHint);
2590 else
2591 return false;
2592
2593 // Convert the comparison and its user to a compare against zero with the
2594 // appropriate predicate on the branch. Zero comparison might provide
2595 // optimization opportunities post-RA (see optimization in
2596 // PPCPreEmitPeephole.cpp).
2597 UseMI->getOperand(i: 0).setImm(Pred);
2598 CmpInstr.getOperand(i: 2).setImm(0);
2599 }
2600
2601 // Search for Sub.
2602 --I;
2603
2604 // Get ready to iterate backward from CmpInstr.
2605 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2606
2607 for (; I != E && !noSub; --I) {
2608 const MachineInstr &Instr = *I;
2609 unsigned IOpC = Instr.getOpcode();
2610
2611 if (&*I != &CmpInstr && (Instr.modifiesRegister(Reg: PPC::CR0, TRI: &RI) ||
2612 Instr.readsRegister(Reg: PPC::CR0, TRI: &RI)))
2613 // This instruction modifies or uses the record condition register after
2614 // the one we want to change. While we could do this transformation, it
2615 // would likely not be profitable. This transformation removes one
2616 // instruction, and so even forcing RA to generate one move probably
2617 // makes it unprofitable.
2618 return false;
2619
2620 // Check whether CmpInstr can be made redundant by the current instruction.
2621 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2622 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2623 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2624 ((Instr.getOperand(i: 1).getReg() == SrcReg &&
2625 Instr.getOperand(i: 2).getReg() == SrcReg2) ||
2626 (Instr.getOperand(i: 1).getReg() == SrcReg2 &&
2627 Instr.getOperand(i: 2).getReg() == SrcReg))) {
2628 Sub = &*I;
2629 break;
2630 }
2631
2632 if (I == B)
2633 // The 'and' is below the comparison instruction.
2634 return false;
2635 }
2636
2637 // Return false if no candidates exist.
2638 if (!MI && !Sub)
2639 return false;
2640
2641 // The single candidate is called MI.
2642 if (!MI) MI = Sub;
2643
2644 int NewOpC = -1;
2645 int MIOpC = MI->getOpcode();
2646 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2647 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2648 NewOpC = MIOpC;
2649 else {
2650 NewOpC = PPC::getRecordFormOpcode(Opcode: MIOpC);
2651 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(Opcode: MIOpC) != -1)
2652 NewOpC = MIOpC;
2653 }
2654
2655 // FIXME: On the non-embedded POWER architectures, only some of the record
2656 // forms are fast, and we should use only the fast ones.
2657
2658 // The defining instruction has a record form (or is already a record
2659 // form). It is possible, however, that we'll need to reverse the condition
2660 // code of the users.
2661 if (NewOpC == -1)
2662 return false;
2663
2664 // Converting MI to its record form makes it define CR0, which is only legal
2665 // if CR0 is dead at that point. CR0 may still be live from an earlier
2666 // record-form instruction with a use scheduled after MI: expanding the
2667 // ANDI_rec_1_{EQ,GT}_BIT pseudos produces COPYs of CR0 subregisters, and
2668 // nothing keeps those adjacent to their `andi.` once other instructions are
2669 // scheduled in between. The backward scan above only covers the range
2670 // between MI and CmpInstr, so it cannot see such a use. Query CR0's
2671 // liveness right after MI, using the block size as the neighborhood so the
2672 // query can inspect the remainder of the block, and be conservative about
2673 // an unknown answer.
2674 const bool IntroducesRecFormInstr = MIOpC != NewOpC;
2675 if (IntroducesRecFormInstr) {
2676 const MachineBasicBlock *MBB = MI->getParent();
2677 if (MBB->computeRegisterLiveness(
2678 TRI: &RI, Reg: PPC::CR0, Before: std::next(x: MachineBasicBlock::const_iterator(MI)),
2679 /*Neighborhood=*/MBB->size()) != MachineBasicBlock::LQR_Dead)
2680 return false;
2681 }
2682
2683 // This transformation should not be performed if `nsw` is missing and is not
2684 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2685 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2686 // CRReg can reflect if compared values are equal, this optz is still valid.
2687 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2688 Sub && !Sub->getFlag(Flag: MachineInstr::NoSWrap))
2689 return false;
2690
2691 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2692 // needs to be updated to be based on SUB. Push the condition code
2693 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2694 // condition code of these operands will be modified.
2695 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2696 // comparison against 0, which may modify predicate.
2697 bool ShouldSwap = false;
2698 if (Sub && Value == 0) {
2699 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(i: 1).getReg() == SrcReg2 &&
2700 Sub->getOperand(i: 2).getReg() == SrcReg;
2701
2702 // The operands to subf are the opposite of sub, so only in the fixed-point
2703 // case, invert the order.
2704 ShouldSwap = !ShouldSwap;
2705 }
2706
2707 if (ShouldSwap)
2708 for (MachineRegisterInfo::use_instr_iterator
2709 I = MRI->use_instr_begin(RegNo: CRReg), IE = MRI->use_instr_end();
2710 I != IE; ++I) {
2711 MachineInstr *UseMI = &*I;
2712 if (UseMI->getOpcode() == PPC::BCC) {
2713 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(i: 0).getImm();
2714 unsigned PredCond = PPC::getPredicateCondition(Opcode: Pred);
2715 assert((!equalityOnly ||
2716 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2717 "Invalid predicate for equality-only optimization");
2718 (void)PredCond; // To suppress warning in release build.
2719 PredsToUpdate.push_back(Elt: std::make_pair(x: &(UseMI->getOperand(i: 0)),
2720 y: PPC::getSwappedPredicate(Opcode: Pred)));
2721 } else if (UseMI->getOpcode() == PPC::ISEL ||
2722 UseMI->getOpcode() == PPC::ISEL8) {
2723 unsigned NewSubReg = UseMI->getOperand(i: 3).getSubReg();
2724 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2725 "Invalid CR bit for equality-only optimization");
2726
2727 if (NewSubReg == PPC::sub_lt)
2728 NewSubReg = PPC::sub_gt;
2729 else if (NewSubReg == PPC::sub_gt)
2730 NewSubReg = PPC::sub_lt;
2731
2732 SubRegsToUpdate.push_back(Elt: std::make_pair(x: &(UseMI->getOperand(i: 3)),
2733 y&: NewSubReg));
2734 } else // We need to abort on a user we don't understand.
2735 return false;
2736 }
2737 assert(!(Value != 0 && ShouldSwap) &&
2738 "Non-zero immediate support and ShouldSwap"
2739 "may conflict in updating predicate");
2740
2741 // Create a new virtual register to hold the value of the CR set by the
2742 // record-form instruction. If the instruction was not previously in
2743 // record form, then set the kill flag on the CR.
2744 CmpInstr.eraseFromParent();
2745
2746 MachineBasicBlock::iterator MII = MI;
2747 BuildMI(BB&: *MI->getParent(), I: std::next(x: MII), MIMD: MI->getDebugLoc(),
2748 MCID: get(Opcode: TargetOpcode::COPY), DestReg: CRReg)
2749 .addReg(RegNo: PPC::CR0, Flags: getKillRegState(B: MIOpC != NewOpC));
2750
2751 // Even if CR0 register were dead before, it is alive now since the
2752 // instruction we just built uses it.
2753 MI->clearRegisterDeads(Reg: PPC::CR0);
2754
2755 if (MIOpC != NewOpC) {
2756 // We need to be careful here: we're replacing one instruction with
2757 // another, and we need to make sure that we get all of the right
2758 // implicit uses and defs. On the other hand, the caller may be holding
2759 // an iterator to this instruction, and so we can't delete it (this is
2760 // specifically the case if this is the instruction directly after the
2761 // compare).
2762
2763 // Rotates are expensive instructions. If we're emitting a record-form
2764 // rotate that can just be an andi/andis, we should just emit that.
2765 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2766 Register GPRRes = MI->getOperand(i: 0).getReg();
2767 int64_t SH = MI->getOperand(i: 2).getImm();
2768 int64_t MB = MI->getOperand(i: 3).getImm();
2769 int64_t ME = MI->getOperand(i: 4).getImm();
2770 // We can only do this if both the start and end of the mask are in the
2771 // same halfword.
2772 bool MBInLoHWord = MB >= 16;
2773 bool MEInLoHWord = ME >= 16;
2774 uint64_t Mask = ~0LLU;
2775
2776 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2777 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2778 // The mask value needs to shift right 16 if we're emitting andis.
2779 Mask >>= MBInLoHWord ? 0 : 16;
2780 NewOpC = MIOpC == PPC::RLWINM
2781 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2782 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2783 } else if (MRI->use_empty(RegNo: GPRRes) && (ME == 31) &&
2784 (ME - MB + 1 == SH) && (MB >= 16)) {
2785 // If we are rotating by the exact number of bits as are in the mask
2786 // and the mask is in the least significant bits of the register,
2787 // that's just an andis. (as long as the GPR result has no uses).
2788 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2789 Mask >>= 16;
2790 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2791 }
2792 // If we've set the mask, we can transform.
2793 if (Mask != ~0LLU) {
2794 MI->removeOperand(OpNo: 4);
2795 MI->removeOperand(OpNo: 3);
2796 MI->getOperand(i: 2).setImm(Mask);
2797 NumRcRotatesConvertedToRcAnd++;
2798 }
2799 } else if (MIOpC == PPC::RLDICL && MI->getOperand(i: 2).getImm() == 0) {
2800 int64_t MB = MI->getOperand(i: 3).getImm();
2801 if (MB >= 48) {
2802 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2803 NewOpC = PPC::ANDI8_rec;
2804 MI->removeOperand(OpNo: 3);
2805 MI->getOperand(i: 2).setImm(Mask);
2806 NumRcRotatesConvertedToRcAnd++;
2807 }
2808 }
2809
2810 const MCInstrDesc &NewDesc = get(Opcode: NewOpC);
2811 MI->setDesc(NewDesc);
2812
2813 for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2814 if (!MI->definesRegister(Reg: ImpDef, /*TRI=*/nullptr)) {
2815 MI->addOperand(MF&: *MI->getParent()->getParent(),
2816 Op: MachineOperand::CreateReg(Reg: ImpDef, isDef: true, isImp: true));
2817 }
2818 }
2819 for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2820 if (!MI->readsRegister(Reg: ImpUse, /*TRI=*/nullptr)) {
2821 MI->addOperand(MF&: *MI->getParent()->getParent(),
2822 Op: MachineOperand::CreateReg(Reg: ImpUse, isDef: false, isImp: true));
2823 }
2824 }
2825 }
2826 assert(MI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
2827 "Record-form instruction does not define cr0?");
2828
2829 // Modify the condition code of operands in OperandsToUpdate.
2830 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2831 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2832 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2833 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2834
2835 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2836 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2837
2838 return true;
2839}
2840
2841bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
2842 MachineRegisterInfo *MRI = &CmpMI.getParent()->getParent()->getRegInfo();
2843 if (MRI->isSSA())
2844 return false;
2845
2846 Register SrcReg, SrcReg2;
2847 int64_t CmpMask, CmpValue;
2848 if (!analyzeCompare(MI: CmpMI, SrcReg, SrcReg2, Mask&: CmpMask, Value&: CmpValue))
2849 return false;
2850
2851 // Try to optimize the comparison against 0.
2852 if (CmpValue || !CmpMask || SrcReg2)
2853 return false;
2854
2855 // The record forms set the condition register based on a signed comparison
2856 // with zero (see comments in optimizeCompareInstr). Since we can't do the
2857 // equality checks in post-RA, we are more restricted on a unsigned
2858 // comparison.
2859 unsigned Opc = CmpMI.getOpcode();
2860 if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2861 return false;
2862
2863 // The record forms are always based on a 64-bit comparison on PPC64
2864 // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2865 // comparison. Since we can't do the equality checks in post-RA, we bail out
2866 // the case.
2867 if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2868 return false;
2869
2870 // CmpMI can't be deleted if it has implicit def.
2871 if (CmpMI.hasImplicitDef())
2872 return false;
2873
2874 bool SrcRegHasOtherUse = false;
2875 MachineInstr *SrcMI = getDefMIPostRA(Reg: SrcReg, MI&: CmpMI, SeenIntermediateUse&: SrcRegHasOtherUse);
2876 if (!SrcMI || !SrcMI->definesRegister(Reg: SrcReg, /*TRI=*/nullptr))
2877 return false;
2878
2879 MachineOperand RegMO = CmpMI.getOperand(i: 0);
2880 Register CRReg = RegMO.getReg();
2881 if (CRReg != PPC::CR0)
2882 return false;
2883
2884 // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2885 bool SeenUseOfCRReg = false;
2886 bool IsCRRegKilled = false;
2887 if (!isRegElgibleForForwarding(RegMO, DefMI: *SrcMI, MI: CmpMI, KillDefMI: false, IsFwdFeederRegKilled&: IsCRRegKilled,
2888 SeenIntermediateUse&: SeenUseOfCRReg) ||
2889 SrcMI->definesRegister(Reg: CRReg, /*TRI=*/nullptr) || SeenUseOfCRReg)
2890 return false;
2891
2892 int SrcMIOpc = SrcMI->getOpcode();
2893 int NewOpC = PPC::getRecordFormOpcode(Opcode: SrcMIOpc);
2894 if (NewOpC == -1)
2895 return false;
2896
2897 LLVM_DEBUG(dbgs() << "Replace Instr: ");
2898 LLVM_DEBUG(SrcMI->dump());
2899
2900 const MCInstrDesc &NewDesc = get(Opcode: NewOpC);
2901 SrcMI->setDesc(NewDesc);
2902 MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2903 .addReg(RegNo: CRReg, Flags: RegState::ImplicitDefine);
2904 SrcMI->clearRegisterDeads(Reg: CRReg);
2905
2906 assert(SrcMI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
2907 "Record-form instruction does not define cr0?");
2908
2909 LLVM_DEBUG(dbgs() << "with: ");
2910 LLVM_DEBUG(SrcMI->dump());
2911 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2912 LLVM_DEBUG(CmpMI.dump());
2913 return true;
2914}
2915
2916bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
2917 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2918 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
2919 const MachineOperand *BaseOp;
2920 OffsetIsScalable = false;
2921 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
2922 return false;
2923 BaseOps.push_back(Elt: BaseOp);
2924 return true;
2925}
2926
2927static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2928 const TargetRegisterInfo *TRI) {
2929 // If this is a volatile load/store, don't mess with it.
2930 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2931 return false;
2932
2933 if (LdSt.getOperand(i: 2).isFI())
2934 return true;
2935
2936 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2937 // Can't cluster if the instruction modifies the base register
2938 // or it is update form. e.g. ld r2,3(r2)
2939 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 2).getReg(), TRI))
2940 return false;
2941
2942 return true;
2943}
2944
2945// Only cluster instruction pair that have the same opcode, and they are
2946// clusterable according to PowerPC specification.
2947static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2948 const PPCSubtarget &Subtarget) {
2949 switch (FirstOpc) {
2950 default:
2951 return false;
2952 case PPC::STD:
2953 case PPC::STFD:
2954 case PPC::STXSD:
2955 case PPC::DFSTOREf64:
2956 return FirstOpc == SecondOpc;
2957 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2958 // 32bit and 64bit instruction selection. They are clusterable pair though
2959 // they are different opcode.
2960 case PPC::STW:
2961 case PPC::STW8:
2962 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2963 }
2964}
2965
2966bool PPCInstrInfo::shouldClusterMemOps(
2967 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
2968 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2969 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
2970 unsigned NumBytes) const {
2971
2972 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2973 const MachineOperand &BaseOp1 = *BaseOps1.front();
2974 const MachineOperand &BaseOp2 = *BaseOps2.front();
2975 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2976 "Only base registers and frame indices are supported.");
2977
2978 // ClusterSize means the number of memory operations that will have been
2979 // clustered if this hook returns true.
2980 // Don't cluster memory op if there are already two ops clustered at least.
2981 if (ClusterSize > 2)
2982 return false;
2983
2984 // Cluster the load/store only when they have the same base
2985 // register or FI.
2986 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2987 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2988 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2989 return false;
2990
2991 // Check if the load/store are clusterable according to the PowerPC
2992 // specification.
2993 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2994 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2995 unsigned FirstOpc = FirstLdSt.getOpcode();
2996 unsigned SecondOpc = SecondLdSt.getOpcode();
2997 // Cluster the load/store only when they have the same opcode, and they are
2998 // clusterable opcode according to PowerPC specification.
2999 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
3000 return false;
3001
3002 // Can't cluster load/store that have ordered or volatile memory reference.
3003 if (!isLdStSafeToCluster(LdSt: FirstLdSt, TRI: &RI) ||
3004 !isLdStSafeToCluster(LdSt: SecondLdSt, TRI: &RI))
3005 return false;
3006
3007 int64_t Offset1 = 0, Offset2 = 0;
3008 LocationSize Width1 = LocationSize::precise(Value: 0),
3009 Width2 = LocationSize::precise(Value: 0);
3010 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
3011 if (!getMemOperandWithOffsetWidth(LdSt: FirstLdSt, BaseOp&: Base1, Offset&: Offset1, Width&: Width1) ||
3012 !getMemOperandWithOffsetWidth(LdSt: SecondLdSt, BaseOp&: Base2, Offset&: Offset2, Width&: Width2) ||
3013 Width1 != Width2)
3014 return false;
3015
3016 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
3017 "getMemOperandWithOffsetWidth return incorrect base op");
3018 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
3019 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
3020 return Offset1 + (int64_t)Width1.getValue() == Offset2;
3021}
3022
3023/// GetInstSize - Return the number of bytes of code the specified
3024/// instruction may be. This returns the maximum number of bytes.
3025///
3026unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
3027 unsigned Opcode = MI.getOpcode();
3028
3029 switch (Opcode) {
3030 case PPC::INLINEASM:
3031 case PPC::INLINEASM_BR: {
3032 const MachineFunction *MF = MI.getParent()->getParent();
3033 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
3034 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo());
3035 }
3036 case TargetOpcode::STACKMAP: {
3037 StackMapOpers Opers(&MI);
3038 return Opers.getNumPatchBytes();
3039 }
3040 case TargetOpcode::PATCHPOINT: {
3041 PatchPointOpers Opers(&MI);
3042 return Opers.getNumPatchBytes();
3043 }
3044 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
3045 const MachineFunction *MF = MI.getParent()->getParent();
3046 const Function &F = MF->getFunction();
3047 unsigned Num = F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
3048 if (Num || MF->getTarget().getTargetTriple().isOSAIX() ||
3049 !MF->getTarget().getTargetTriple().isLittleEndian())
3050 return Num * 4;
3051 // Size of xray sled.
3052 return 7 * 4;
3053 }
3054 case TargetOpcode::PATCHABLE_RET: {
3055 // Size of xray sled.
3056 unsigned RetOpcode = MI.getOperand(i: 0).getImm();
3057 bool IsConditional = RetOpcode == PPC::BCCLR;
3058 return (8 + IsConditional) * 4;
3059 }
3060 case TargetOpcode::BUNDLE:
3061 return getInstBundleSize(MI);
3062 default:
3063 return get(Opcode).getSize();
3064 }
3065}
3066
3067TargetInstrInfo::InstSizeVerifyMode
3068PPCInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
3069 // FIXME: The size of STACKMAP is currently over-estimated.
3070 return MI.getOpcode() == TargetOpcode::STACKMAP
3071 ? InstSizeVerifyMode::AllowOverEstimate
3072 : InstSizeVerifyMode::ExactSize;
3073}
3074
3075std::pair<unsigned, unsigned>
3076PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3077 // PPC always uses a direct mask.
3078 return std::make_pair(x&: TF, y: 0u);
3079}
3080
3081ArrayRef<std::pair<unsigned, const char *>>
3082PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3083 using namespace PPCII;
3084 static const std::pair<unsigned, const char *> TargetFlags[] = {
3085 {MO_PLT, "ppc-plt"},
3086 {MO_PIC_FLAG, "ppc-pic"},
3087 {MO_PCREL_FLAG, "ppc-pcrel"},
3088 {MO_GOT_FLAG, "ppc-got"},
3089 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
3090 {MO_TLSGD_FLAG, "ppc-tlsgd"},
3091 {MO_TPREL_FLAG, "ppc-tprel"},
3092 {MO_TLSLDM_FLAG, "ppc-tlsldm"},
3093 {MO_TLSLD_FLAG, "ppc-tlsld"},
3094 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
3095 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
3096 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
3097 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"},
3098 {MO_LO, "ppc-lo"},
3099 {MO_HA, "ppc-ha"},
3100 {MO_TPREL_LO, "ppc-tprel-lo"},
3101 {MO_TPREL_HA, "ppc-tprel-ha"},
3102 {MO_DTPREL_LO, "ppc-dtprel-lo"},
3103 {MO_TLSLD_LO, "ppc-tlsld-lo"},
3104 {MO_TOC_LO, "ppc-toc-lo"},
3105 {MO_TLS, "ppc-tls"},
3106 {MO_PIC_HA_FLAG, "ppc-ha-pic"},
3107 {MO_PIC_LO_FLAG, "ppc-lo-pic"},
3108 {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"},
3109 {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"},
3110 {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"},
3111 };
3112 return ArrayRef(TargetFlags);
3113}
3114
3115// Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
3116// The VSX versions have the advantage of a full 64-register target whereas
3117// the FP ones have the advantage of lower latency and higher throughput. So
3118// what we are after is using the faster instructions in low register pressure
3119// situations and using the larger register file in high register pressure
3120// situations.
3121bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
3122 unsigned UpperOpcode, LowerOpcode;
3123 switch (MI.getOpcode()) {
3124 case PPC::DFLOADf32:
3125 UpperOpcode = PPC::LXSSP;
3126 LowerOpcode = PPC::LFS;
3127 break;
3128 case PPC::DFLOADf64:
3129 UpperOpcode = PPC::LXSD;
3130 LowerOpcode = PPC::LFD;
3131 break;
3132 case PPC::DFSTOREf32:
3133 UpperOpcode = PPC::STXSSP;
3134 LowerOpcode = PPC::STFS;
3135 break;
3136 case PPC::DFSTOREf64:
3137 UpperOpcode = PPC::STXSD;
3138 LowerOpcode = PPC::STFD;
3139 break;
3140 case PPC::XFLOADf32:
3141 UpperOpcode = PPC::LXSSPX;
3142 LowerOpcode = PPC::LFSX;
3143 break;
3144 case PPC::XFLOADf64:
3145 UpperOpcode = PPC::LXSDX;
3146 LowerOpcode = PPC::LFDX;
3147 break;
3148 case PPC::XFSTOREf32:
3149 UpperOpcode = PPC::STXSSPX;
3150 LowerOpcode = PPC::STFSX;
3151 break;
3152 case PPC::XFSTOREf64:
3153 UpperOpcode = PPC::STXSDX;
3154 LowerOpcode = PPC::STFDX;
3155 break;
3156 case PPC::LIWAX:
3157 UpperOpcode = PPC::LXSIWAX;
3158 LowerOpcode = PPC::LFIWAX;
3159 break;
3160 case PPC::LIWZX:
3161 UpperOpcode = PPC::LXSIWZX;
3162 LowerOpcode = PPC::LFIWZX;
3163 break;
3164 case PPC::STIWX:
3165 UpperOpcode = PPC::STXSIWX;
3166 LowerOpcode = PPC::STFIWX;
3167 break;
3168 default:
3169 llvm_unreachable("Unknown Operation!");
3170 }
3171
3172 Register TargetReg = MI.getOperand(i: 0).getReg();
3173 unsigned Opcode;
3174 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3175 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3176 Opcode = LowerOpcode;
3177 else
3178 Opcode = UpperOpcode;
3179 MI.setDesc(get(Opcode));
3180 return true;
3181}
3182
3183static bool isAnImmediateOperand(const MachineOperand &MO) {
3184 return MO.isCPI() || MO.isGlobal() || MO.isImm();
3185}
3186
3187bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
3188 auto &MBB = *MI.getParent();
3189 auto DL = MI.getDebugLoc();
3190
3191 switch (MI.getOpcode()) {
3192 case PPC::BUILD_UACC: {
3193 MCRegister ACC = MI.getOperand(i: 0).getReg();
3194 MCRegister UACC = MI.getOperand(i: 1).getReg();
3195 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3196 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3197 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3198 // FIXME: This can easily be improved to look up to the top of the MBB
3199 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3200 // we can just re-target any such XXLOR's to DstVSR + offset.
3201 for (int VecNo = 0; VecNo < 4; VecNo++)
3202 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::XXLOR), DestReg: DstVSR + VecNo)
3203 .addReg(RegNo: SrcVSR + VecNo)
3204 .addReg(RegNo: SrcVSR + VecNo);
3205 }
3206 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3207 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3208 // with a NOP.
3209 [[fallthrough]];
3210 }
3211 case PPC::KILL_PAIR: {
3212 MI.setDesc(get(Opcode: PPC::UNENCODED_NOP));
3213 MI.removeOperand(OpNo: 1);
3214 MI.removeOperand(OpNo: 0);
3215 return true;
3216 }
3217 case TargetOpcode::LOAD_STACK_GUARD: {
3218 auto M = MBB.getParent()->getFunction().getParent();
3219 assert(
3220 (Subtarget.isTargetLinux() || M->getStackProtectorGuard() == "tls") &&
3221 "Only Linux target or tls mode are expected to contain "
3222 "LOAD_STACK_GUARD");
3223 int64_t Offset;
3224 if (M->getStackProtectorGuard() == "tls")
3225 Offset = M->getStackProtectorGuardOffset();
3226 else
3227 Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3228 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3229 MI.setDesc(get(Opcode: Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3230 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3231 .addImm(Val: Offset)
3232 .addReg(RegNo: Reg);
3233 return true;
3234 }
3235 case PPC::PPCLdFixedAddr: {
3236 assert((Subtarget.getTargetTriple().isOSGlibc() ||
3237 Subtarget.getTargetTriple().isMusl()) &&
3238 "Only targets with Glibc expected to contain PPCLdFixedAddr");
3239 int64_t Offset = 0;
3240 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3241 MI.setDesc(get(Opcode: PPC::LWZ));
3242 uint64_t FAType = MI.getOperand(i: 1).getImm();
3243#undef PPC_LNX_FEATURE
3244#undef PPC_CPU
3245#define PPC_LNX_DEFINE_OFFSETS
3246#include "llvm/TargetParser/PPCTargetParser.def"
3247 bool IsLE = Subtarget.isLittleEndian();
3248 bool Is64 = Subtarget.isPPC64();
3249 if (FAType == PPC_FAWORD_HWCAP) {
3250 if (IsLE)
3251 Offset = Is64 ? PPC_HWCAP_OFFSET_LE64 : PPC_HWCAP_OFFSET_LE32;
3252 else
3253 Offset = Is64 ? PPC_HWCAP_OFFSET_BE64 : PPC_HWCAP_OFFSET_BE32;
3254 } else if (FAType == PPC_FAWORD_HWCAP2) {
3255 if (IsLE)
3256 Offset = Is64 ? PPC_HWCAP2_OFFSET_LE64 : PPC_HWCAP2_OFFSET_LE32;
3257 else
3258 Offset = Is64 ? PPC_HWCAP2_OFFSET_BE64 : PPC_HWCAP2_OFFSET_BE32;
3259 } else if (FAType == PPC_FAWORD_CPUID) {
3260 if (IsLE)
3261 Offset = Is64 ? PPC_CPUID_OFFSET_LE64 : PPC_CPUID_OFFSET_LE32;
3262 else
3263 Offset = Is64 ? PPC_CPUID_OFFSET_BE64 : PPC_CPUID_OFFSET_BE32;
3264 }
3265 assert(Offset && "Do not know the offset for this fixed addr load");
3266 MI.removeOperand(OpNo: 1);
3267 Subtarget.getTargetMachine().setGlibcHWCAPAccess();
3268 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3269 .addImm(Val: Offset)
3270 .addReg(RegNo: Reg);
3271 return true;
3272#define PPC_TGT_PARSER_UNDEF_MACROS
3273#include "llvm/TargetParser/PPCTargetParser.def"
3274#undef PPC_TGT_PARSER_UNDEF_MACROS
3275 }
3276 case PPC::DFLOADf32:
3277 case PPC::DFLOADf64:
3278 case PPC::DFSTOREf32:
3279 case PPC::DFSTOREf64: {
3280 assert(Subtarget.hasP9Vector() &&
3281 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3282 assert(MI.getOperand(2).isReg() &&
3283 isAnImmediateOperand(MI.getOperand(1)) &&
3284 "D-form op must have register and immediate operands");
3285 return expandVSXMemPseudo(MI);
3286 }
3287 case PPC::XFLOADf32:
3288 case PPC::XFSTOREf32:
3289 case PPC::LIWAX:
3290 case PPC::LIWZX:
3291 case PPC::STIWX: {
3292 assert(Subtarget.hasP8Vector() &&
3293 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3294 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3295 "X-form op must have register and register operands");
3296 return expandVSXMemPseudo(MI);
3297 }
3298 case PPC::XFLOADf64:
3299 case PPC::XFSTOREf64: {
3300 assert(Subtarget.hasVSX() &&
3301 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3302 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3303 "X-form op must have register and register operands");
3304 return expandVSXMemPseudo(MI);
3305 }
3306 case PPC::SPILLTOVSR_LD: {
3307 Register TargetReg = MI.getOperand(i: 0).getReg();
3308 if (PPC::VSFRCRegClass.contains(Reg: TargetReg)) {
3309 MI.setDesc(get(Opcode: PPC::DFLOADf64));
3310 return expandPostRAPseudo(MI);
3311 }
3312 else
3313 MI.setDesc(get(Opcode: PPC::LD));
3314 return true;
3315 }
3316 case PPC::SPILLTOVSR_ST: {
3317 Register SrcReg = MI.getOperand(i: 0).getReg();
3318 if (PPC::VSFRCRegClass.contains(Reg: SrcReg)) {
3319 NumStoreSPILLVSRRCAsVec++;
3320 MI.setDesc(get(Opcode: PPC::DFSTOREf64));
3321 return expandPostRAPseudo(MI);
3322 } else {
3323 NumStoreSPILLVSRRCAsGpr++;
3324 MI.setDesc(get(Opcode: PPC::STD));
3325 }
3326 return true;
3327 }
3328 case PPC::SPILLTOVSR_LDX: {
3329 Register TargetReg = MI.getOperand(i: 0).getReg();
3330 if (PPC::VSFRCRegClass.contains(Reg: TargetReg))
3331 MI.setDesc(get(Opcode: PPC::LXSDX));
3332 else
3333 MI.setDesc(get(Opcode: PPC::LDX));
3334 return true;
3335 }
3336 case PPC::SPILLTOVSR_STX: {
3337 Register SrcReg = MI.getOperand(i: 0).getReg();
3338 if (PPC::VSFRCRegClass.contains(Reg: SrcReg)) {
3339 NumStoreSPILLVSRRCAsVec++;
3340 MI.setDesc(get(Opcode: PPC::STXSDX));
3341 } else {
3342 NumStoreSPILLVSRRCAsGpr++;
3343 MI.setDesc(get(Opcode: PPC::STDX));
3344 }
3345 return true;
3346 }
3347
3348 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3349 case PPC::CFENCE:
3350 case PPC::CFENCE8: {
3351 auto Val = MI.getOperand(i: 0).getReg();
3352 unsigned CmpOp = Subtarget.isPPC64() ? PPC::CMPD : PPC::CMPW;
3353 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: CmpOp), DestReg: PPC::CR7).addReg(RegNo: Val).addReg(RegNo: Val);
3354 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::CTRL_DEP))
3355 .addImm(Val: PPC::PRED_NE_MINUS)
3356 .addReg(RegNo: PPC::CR7)
3357 .addImm(Val: 1);
3358 MI.setDesc(get(Opcode: PPC::ISYNC));
3359 MI.removeOperand(OpNo: 0);
3360 return true;
3361 }
3362 case PPC::LWAT_CSNE_PSEUDO:
3363 case PPC::LDAT_CSNE_PSEUDO:
3364 return expandAMOCSNEPseudo(MI);
3365 }
3366 return false;
3367}
3368
3369// Essentially a compile-time implementation of a compare->isel sequence.
3370// It takes two constants to compare, along with the true/false registers
3371// and the comparison type (as a subreg to a CR field) and returns one
3372// of the true/false registers, depending on the comparison results.
3373static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3374 unsigned TrueReg, unsigned FalseReg,
3375 unsigned CRSubReg) {
3376 // Signed comparisons. The immediates are assumed to be sign-extended.
3377 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3378 switch (CRSubReg) {
3379 default: llvm_unreachable("Unknown integer comparison type.");
3380 case PPC::sub_lt:
3381 return Imm1 < Imm2 ? TrueReg : FalseReg;
3382 case PPC::sub_gt:
3383 return Imm1 > Imm2 ? TrueReg : FalseReg;
3384 case PPC::sub_eq:
3385 return Imm1 == Imm2 ? TrueReg : FalseReg;
3386 }
3387 }
3388 // Unsigned comparisons.
3389 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3390 switch (CRSubReg) {
3391 default: llvm_unreachable("Unknown integer comparison type.");
3392 case PPC::sub_lt:
3393 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3394 case PPC::sub_gt:
3395 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3396 case PPC::sub_eq:
3397 return Imm1 == Imm2 ? TrueReg : FalseReg;
3398 }
3399 }
3400 return PPC::NoRegister;
3401}
3402
3403void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
3404 unsigned OpNo,
3405 int64_t Imm) const {
3406 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3407 // Replace the REG with the Immediate.
3408 Register InUseReg = MI.getOperand(i: OpNo).getReg();
3409 MI.getOperand(i: OpNo).ChangeToImmediate(ImmVal: Imm);
3410
3411 // We need to make sure that the MI didn't have any implicit use
3412 // of this REG any more. We don't call MI.implicit_operands().empty() to
3413 // return early, since MI's MCID might be changed in calling context, as a
3414 // result its number of explicit operands may be changed, thus the begin of
3415 // implicit operand is changed.
3416 int UseOpIdx = MI.findRegisterUseOperandIdx(Reg: InUseReg, TRI: &RI, isKill: false);
3417 if (UseOpIdx >= 0) {
3418 MachineOperand &MO = MI.getOperand(i: UseOpIdx);
3419 if (MO.isImplicit())
3420 // The operands must always be in the following order:
3421 // - explicit reg defs,
3422 // - other explicit operands (reg uses, immediates, etc.),
3423 // - implicit reg defs
3424 // - implicit reg uses
3425 // Therefore, removing the implicit operand won't change the explicit
3426 // operands layout.
3427 MI.removeOperand(OpNo: UseOpIdx);
3428 }
3429}
3430
3431// Replace an instruction with one that materializes a constant (and sets
3432// CR0 if the original instruction was a record-form instruction).
3433void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
3434 const LoadImmediateInfo &LII) const {
3435 // Remove existing operands.
3436 int OperandToKeep = LII.SetCR ? 1 : 0;
3437 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3438 MI.removeOperand(OpNo: i);
3439
3440 // Replace the instruction.
3441 if (LII.SetCR) {
3442 MI.setDesc(get(Opcode: LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3443 // Set the immediate.
3444 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3445 .addImm(Val: LII.Imm).addReg(RegNo: PPC::CR0, Flags: RegState::ImplicitDefine);
3446 return;
3447 }
3448 else
3449 MI.setDesc(get(Opcode: LII.Is64Bit ? PPC::LI8 : PPC::LI));
3450
3451 // Set the immediate.
3452 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3453 .addImm(Val: LII.Imm);
3454}
3455
3456MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
3457 bool &SeenIntermediateUse) const {
3458 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3459 "Should be called after register allocation.");
3460 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3461 It++;
3462 SeenIntermediateUse = false;
3463 for (; It != E; ++It) {
3464 if (It->modifiesRegister(Reg, TRI: &RI))
3465 return &*It;
3466 if (It->readsRegister(Reg, TRI: &RI))
3467 SeenIntermediateUse = true;
3468 }
3469 return nullptr;
3470}
3471
3472void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB,
3473 MachineBasicBlock::iterator MBBI,
3474 const DebugLoc &DL, Register Reg,
3475 int64_t Imm) const {
3476 assert(!MBB.getParent()->getRegInfo().isSSA() &&
3477 "Register should be in non-SSA form after RA");
3478 bool isPPC64 = Subtarget.isPPC64();
3479 // FIXME: Materialization here is not optimal.
3480 // For some special bit patterns we can use less instructions.
3481 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3482 if (isInt<16>(x: Imm)) {
3483 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: Reg).addImm(Val: Imm);
3484 } else if (isInt<32>(x: Imm)) {
3485 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: Reg)
3486 .addImm(Val: Imm >> 16);
3487 if (Imm & 0xFFFF)
3488 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI), DestReg: Reg)
3489 .addReg(RegNo: Reg, Flags: RegState::Kill)
3490 .addImm(Val: Imm & 0xFFFF);
3491 } else {
3492 assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3493 "only supported in PPC64");
3494 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::LIS8), DestReg: Reg).addImm(Val: Imm >> 48);
3495 if ((Imm >> 32) & 0xFFFF)
3496 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORI8), DestReg: Reg)
3497 .addReg(RegNo: Reg, Flags: RegState::Kill)
3498 .addImm(Val: (Imm >> 32) & 0xFFFF);
3499 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::RLDICR), DestReg: Reg)
3500 .addReg(RegNo: Reg, Flags: RegState::Kill)
3501 .addImm(Val: 32)
3502 .addImm(Val: 31);
3503 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORIS8), DestReg: Reg)
3504 .addReg(RegNo: Reg, Flags: RegState::Kill)
3505 .addImm(Val: (Imm >> 16) & 0xFFFF);
3506 if (Imm & 0xFFFF)
3507 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: PPC::ORI8), DestReg: Reg)
3508 .addReg(RegNo: Reg, Flags: RegState::Kill)
3509 .addImm(Val: Imm & 0xFFFF);
3510 }
3511}
3512
3513MachineInstr *PPCInstrInfo::getForwardingDefMI(
3514 MachineInstr &MI,
3515 unsigned &OpNoForForwarding,
3516 bool &SeenIntermediateUse) const {
3517 OpNoForForwarding = ~0U;
3518 MachineInstr *DefMI = nullptr;
3519 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3520 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3521 // within the basic block to see if the register is defined using an
3522 // LI/LI8/ADDI/ADDI8.
3523 if (MRI->isSSA()) {
3524 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3525 if (!MI.getOperand(i).isReg())
3526 continue;
3527 Register Reg = MI.getOperand(i).getReg();
3528 if (!Reg.isVirtual())
3529 continue;
3530 Register TrueReg = RI.lookThruCopyLike(SrcReg: Reg, MRI);
3531 if (TrueReg.isVirtual()) {
3532 MachineInstr *DefMIForTrueReg = MRI->getVRegDef(Reg: TrueReg);
3533 if (!DefMIForTrueReg)
3534 continue;
3535 if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3536 DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3537 DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3538 DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3539 OpNoForForwarding = i;
3540 DefMI = DefMIForTrueReg;
3541 // The ADDI and LI operand maybe exist in one instruction at same
3542 // time. we prefer to fold LI operand as LI only has one Imm operand
3543 // and is more possible to be converted. So if current DefMI is
3544 // ADDI/ADDI8, we continue to find possible LI/LI8.
3545 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3546 break;
3547 }
3548 }
3549 }
3550 } else {
3551 // Looking back through the definition for each operand could be expensive,
3552 // so exit early if this isn't an instruction that either has an immediate
3553 // form or is already an immediate form that we can handle.
3554 ImmInstrInfo III;
3555 unsigned Opc = MI.getOpcode();
3556 bool ConvertibleImmForm =
3557 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3558 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3559 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3560 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3561 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3562 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3563 Opc == PPC::RLWINM8_rec;
3564 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(i: 0).isReg())
3565 ? PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg())
3566 : false;
3567 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, PostRA: true))
3568 return nullptr;
3569
3570 // Don't convert or %X, %Y, %Y since that's just a register move.
3571 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3572 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
3573 return nullptr;
3574 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3575 MachineOperand &MO = MI.getOperand(i);
3576 SeenIntermediateUse = false;
3577 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3578 Register Reg = MI.getOperand(i).getReg();
3579 // If we see another use of this reg between the def and the MI,
3580 // we want to flag it so the def isn't deleted.
3581 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3582 if (DefMI) {
3583 // Is this register defined by some form of add-immediate (including
3584 // load-immediate) within this basic block?
3585 switch (DefMI->getOpcode()) {
3586 default:
3587 break;
3588 case PPC::LI:
3589 case PPC::LI8:
3590 case PPC::ADDItocL8:
3591 case PPC::ADDI:
3592 case PPC::ADDI8:
3593 OpNoForForwarding = i;
3594 return DefMI;
3595 }
3596 }
3597 }
3598 }
3599 }
3600 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3601}
3602
3603unsigned PPCInstrInfo::getSpillTarget() const {
3604 // With P10, we may need to spill paired vector registers or accumulator
3605 // registers. MMA implies paired vectors, so we can just check that.
3606 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3607 // P11 uses the P10 target.
3608 return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3609 2 : Subtarget.hasP9Vector() ?
3610 1 : 0;
3611}
3612
3613ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3614 return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3615}
3616
3617ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3618 return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3619}
3620
3621// This opt tries to convert the following imm form to an index form to save an
3622// add for stack variables.
3623// Return false if no such pattern found.
3624//
3625// ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3626// ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3627// Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3628//
3629// can be converted to:
3630//
3631// new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3632// Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3633//
3634// In order to eliminate ADD instr, make sure that:
3635// 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3636// new ADDI instr and ADDI can only take int16 Imm.
3637// 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3638// between ADDI and ADD instr since its original def in ADDI will be changed
3639// in new ADDI instr. And also there should be no new def for it between
3640// ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3641// 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3642// between ADD and Imm instr since ADD instr will be eliminated.
3643// 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3644// moved to Index instr.
3645bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
3646 MachineFunction *MF = MI.getParent()->getParent();
3647 MachineRegisterInfo *MRI = &MF->getRegInfo();
3648 bool PostRA = !MRI->isSSA();
3649 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3650 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3651 // frame base(OffsetAddi) are determined.
3652 if (!PostRA)
3653 return false;
3654 unsigned ToBeDeletedReg = 0;
3655 int64_t OffsetImm = 0;
3656 unsigned XFormOpcode = 0;
3657 ImmInstrInfo III;
3658
3659 // Check if Imm instr meets requirement.
3660 if (!isImmInstrEligibleForFolding(MI, BaseReg&: ToBeDeletedReg, XFormOpcode, OffsetOfImmInstr&: OffsetImm,
3661 III))
3662 return false;
3663
3664 bool OtherIntermediateUse = false;
3665 MachineInstr *ADDMI = getDefMIPostRA(Reg: ToBeDeletedReg, MI, SeenIntermediateUse&: OtherIntermediateUse);
3666
3667 // Exit if there is other use between ADD and Imm instr or no def found.
3668 if (OtherIntermediateUse || !ADDMI)
3669 return false;
3670
3671 // Check if ADD instr meets requirement.
3672 if (!isADDInstrEligibleForFolding(ADDMI&: *ADDMI))
3673 return false;
3674
3675 unsigned ScaleRegIdx = 0;
3676 int64_t OffsetAddi = 0;
3677 MachineInstr *ADDIMI = nullptr;
3678
3679 // Check if there is a valid ToBeChangedReg in ADDMI.
3680 // 1: It must be killed.
3681 // 2: Its definition must be a valid ADDIMI.
3682 // 3: It must satify int16 offset requirement.
3683 if (isValidToBeChangedReg(ADDMI, Index: 1, ADDIMI, OffsetAddi, OffsetImm))
3684 ScaleRegIdx = 2;
3685 else if (isValidToBeChangedReg(ADDMI, Index: 2, ADDIMI, OffsetAddi, OffsetImm))
3686 ScaleRegIdx = 1;
3687 else
3688 return false;
3689
3690 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3691 Register ToBeChangedReg = ADDIMI->getOperand(i: 0).getReg();
3692 Register ScaleReg = ADDMI->getOperand(i: ScaleRegIdx).getReg();
3693 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3694 MachineBasicBlock::iterator End) {
3695 for (auto It = ++Start; It != End; It++)
3696 if (It->modifiesRegister(Reg, TRI: &getRegisterInfo()))
3697 return true;
3698 return false;
3699 };
3700
3701 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3702 // treated as special zero when ScaleReg is R0/X0 register.
3703 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3704 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3705 return false;
3706
3707 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3708 // and Imm Instr.
3709 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3710 return false;
3711
3712 // Now start to do the transformation.
3713 LLVM_DEBUG(dbgs() << "Replace instruction: "
3714 << "\n");
3715 LLVM_DEBUG(ADDIMI->dump());
3716 LLVM_DEBUG(ADDMI->dump());
3717 LLVM_DEBUG(MI.dump());
3718 LLVM_DEBUG(dbgs() << "with: "
3719 << "\n");
3720
3721 // Update ADDI instr.
3722 ADDIMI->getOperand(i: 2).setImm(OffsetAddi + OffsetImm);
3723
3724 // Update Imm instr.
3725 MI.setDesc(get(Opcode: XFormOpcode));
3726 MI.getOperand(i: III.ImmOpNo)
3727 .ChangeToRegister(Reg: ScaleReg, isDef: false, isImp: false,
3728 isKill: ADDMI->getOperand(i: ScaleRegIdx).isKill());
3729
3730 MI.getOperand(i: III.OpNoForForwarding)
3731 .ChangeToRegister(Reg: ToBeChangedReg, isDef: false, isImp: false, isKill: true);
3732
3733 // Eliminate ADD instr.
3734 ADDMI->eraseFromParent();
3735
3736 LLVM_DEBUG(ADDIMI->dump());
3737 LLVM_DEBUG(MI.dump());
3738
3739 return true;
3740}
3741
3742bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
3743 int64_t &Imm) const {
3744 unsigned Opc = ADDIMI.getOpcode();
3745
3746 // Exit if the instruction is not ADDI.
3747 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3748 return false;
3749
3750 // The operand may not necessarily be an immediate - it could be a relocation.
3751 if (!ADDIMI.getOperand(i: 2).isImm())
3752 return false;
3753
3754 Imm = ADDIMI.getOperand(i: 2).getImm();
3755
3756 return true;
3757}
3758
3759bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
3760 unsigned Opc = ADDMI.getOpcode();
3761
3762 // Exit if the instruction is not ADD.
3763 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3764}
3765
3766bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
3767 unsigned &ToBeDeletedReg,
3768 unsigned &XFormOpcode,
3769 int64_t &OffsetImm,
3770 ImmInstrInfo &III) const {
3771 // Only handle load/store.
3772 if (!MI.mayLoadOrStore())
3773 return false;
3774
3775 unsigned Opc = MI.getOpcode();
3776
3777 XFormOpcode = RI.getMappedIdxOpcForImmOpc(ImmOpcode: Opc);
3778
3779 // Exit if instruction has no index form.
3780 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3781 return false;
3782
3783 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3784 if (!instrHasImmForm(Opc: XFormOpcode,
3785 IsVFReg: PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg()), III, PostRA: true))
3786 return false;
3787
3788 if (!III.IsSummingOperands)
3789 return false;
3790
3791 MachineOperand ImmOperand = MI.getOperand(i: III.ImmOpNo);
3792 MachineOperand RegOperand = MI.getOperand(i: III.OpNoForForwarding);
3793 // Only support imm operands, not relocation slots or others.
3794 if (!ImmOperand.isImm())
3795 return false;
3796
3797 assert(RegOperand.isReg() && "Instruction format is not right");
3798
3799 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3800 if (!RegOperand.isKill())
3801 return false;
3802
3803 ToBeDeletedReg = RegOperand.getReg();
3804 OffsetImm = ImmOperand.getImm();
3805
3806 return true;
3807}
3808
3809bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
3810 MachineInstr *&ADDIMI,
3811 int64_t &OffsetAddi,
3812 int64_t OffsetImm) const {
3813 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3814 MachineOperand &MO = ADDMI->getOperand(i: Index);
3815
3816 if (!MO.isKill())
3817 return false;
3818
3819 bool OtherIntermediateUse = false;
3820
3821 ADDIMI = getDefMIPostRA(Reg: MO.getReg(), MI&: *ADDMI, SeenIntermediateUse&: OtherIntermediateUse);
3822 // Currently handle only one "add + Imminstr" pair case, exit if other
3823 // intermediate use for ToBeChangedReg found.
3824 // TODO: handle the cases where there are other "add + Imminstr" pairs
3825 // with same offset in Imminstr which is like:
3826 //
3827 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3828 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3829 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3830 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3831 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3832 //
3833 // can be converted to:
3834 //
3835 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3836 // (OffsetAddi + OffsetImm)
3837 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3838 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3839
3840 if (OtherIntermediateUse || !ADDIMI)
3841 return false;
3842 // Check if ADDI instr meets requirement.
3843 if (!isADDIInstrEligibleForFolding(ADDIMI&: *ADDIMI, Imm&: OffsetAddi))
3844 return false;
3845
3846 if (isInt<16>(x: OffsetAddi + OffsetImm))
3847 return true;
3848 return false;
3849}
3850
3851// If this instruction has an immediate form and one of its operands is a
3852// result of a load-immediate or an add-immediate, convert it to
3853// the immediate form if the constant is in range.
3854bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
3855 SmallSet<Register, 4> &RegsToUpdate,
3856 MachineInstr **KilledDef) const {
3857 MachineFunction *MF = MI.getParent()->getParent();
3858 MachineRegisterInfo *MRI = &MF->getRegInfo();
3859 bool PostRA = !MRI->isSSA();
3860 bool SeenIntermediateUse = true;
3861 unsigned ForwardingOperand = ~0U;
3862 MachineInstr *DefMI = getForwardingDefMI(MI, OpNoForForwarding&: ForwardingOperand,
3863 SeenIntermediateUse);
3864 if (!DefMI)
3865 return false;
3866 assert(ForwardingOperand < MI.getNumOperands() &&
3867 "The forwarding operand needs to be valid at this point");
3868 bool IsForwardingOperandKilled = MI.getOperand(i: ForwardingOperand).isKill();
3869 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3870 if (KilledDef && KillFwdDefMI)
3871 *KilledDef = DefMI;
3872
3873 // Conservatively add defs from DefMI and defs/uses from MI to the set of
3874 // registers that need their kill flags updated.
3875 for (const MachineOperand &MO : DefMI->operands())
3876 if (MO.isReg() && MO.isDef())
3877 RegsToUpdate.insert(V: MO.getReg());
3878 for (const MachineOperand &MO : MI.operands())
3879 if (MO.isReg())
3880 RegsToUpdate.insert(V: MO.getReg());
3881
3882 // If this is a imm instruction and its register operands is produced by ADDI,
3883 // put the imm into imm inst directly.
3884 if (RI.getMappedIdxOpcForImmOpc(ImmOpcode: MI.getOpcode()) !=
3885 PPC::INSTRUCTION_LIST_END &&
3886 transformToNewImmFormFedByAdd(MI, DefMI&: *DefMI, OpNoForForwarding: ForwardingOperand))
3887 return true;
3888
3889 ImmInstrInfo III;
3890 bool IsVFReg = MI.getOperand(i: 0).isReg() &&
3891 MI.getOperand(i: 0).getReg().isPhysical() &&
3892 PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg());
3893 bool HasImmForm = instrHasImmForm(Opc: MI.getOpcode(), IsVFReg, III, PostRA);
3894 // If this is a reg+reg instruction that has a reg+imm form,
3895 // and one of the operands is produced by an add-immediate,
3896 // try to convert it.
3897 if (HasImmForm &&
3898 transformToImmFormFedByAdd(MI, III, ConstantOpNo: ForwardingOperand, DefMI&: *DefMI,
3899 KillDefMI: KillFwdDefMI))
3900 return true;
3901
3902 // If this is a reg+reg instruction that has a reg+imm form,
3903 // and one of the operands is produced by LI, convert it now.
3904 if (HasImmForm &&
3905 transformToImmFormFedByLI(MI, III, ConstantOpNo: ForwardingOperand, DefMI&: *DefMI))
3906 return true;
3907
3908 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3909 // can be simpified to LI.
3910 if (!HasImmForm &&
3911 simplifyToLI(MI, DefMI&: *DefMI, OpNoForForwarding: ForwardingOperand, KilledDef, RegsToUpdate: &RegsToUpdate))
3912 return true;
3913
3914 return false;
3915}
3916
3917bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
3918 MachineInstr **ToErase) const {
3919 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3920 Register FoldingReg = MI.getOperand(i: 1).getReg();
3921 if (!FoldingReg.isVirtual())
3922 return false;
3923 MachineInstr *SrcMI = MRI->getVRegDef(Reg: FoldingReg);
3924 if (!SrcMI)
3925 return false;
3926 if (SrcMI->getOpcode() != PPC::RLWINM &&
3927 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3928 SrcMI->getOpcode() != PPC::RLWINM8 &&
3929 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3930 return false;
3931 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3932 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3933 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3934 "Invalid PPC::RLWINM Instruction!");
3935 uint64_t SHSrc = SrcMI->getOperand(i: 2).getImm();
3936 uint64_t SHMI = MI.getOperand(i: 2).getImm();
3937 uint64_t MBSrc = SrcMI->getOperand(i: 3).getImm();
3938 uint64_t MBMI = MI.getOperand(i: 3).getImm();
3939 uint64_t MESrc = SrcMI->getOperand(i: 4).getImm();
3940 uint64_t MEMI = MI.getOperand(i: 4).getImm();
3941
3942 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3943 "Invalid PPC::RLWINM Instruction!");
3944 // If MBMI is bigger than MEMI, we always can not get run of ones.
3945 // RotatedSrcMask non-wrap:
3946 // 0........31|32........63
3947 // RotatedSrcMask: B---E B---E
3948 // MaskMI: -----------|--E B------
3949 // Result: ----- --- (Bad candidate)
3950 //
3951 // RotatedSrcMask wrap:
3952 // 0........31|32........63
3953 // RotatedSrcMask: --E B----|--E B----
3954 // MaskMI: -----------|--E B------
3955 // Result: --- -----|--- ----- (Bad candidate)
3956 //
3957 // One special case is RotatedSrcMask is a full set mask.
3958 // RotatedSrcMask full:
3959 // 0........31|32........63
3960 // RotatedSrcMask: ------EB---|-------EB---
3961 // MaskMI: -----------|--E B------
3962 // Result: -----------|--- ------- (Good candidate)
3963
3964 // Mark special case.
3965 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3966
3967 // For other MBMI > MEMI cases, just return.
3968 if ((MBMI > MEMI) && !SrcMaskFull)
3969 return false;
3970
3971 // Handle MBMI <= MEMI cases.
3972 APInt MaskMI = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - MEMI - 1, hiBit: 32 - MBMI);
3973 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3974 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3975 // while in PowerPC ISA, lowerest bit is at index 63.
3976 APInt MaskSrc = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - MESrc - 1, hiBit: 32 - MBSrc);
3977
3978 APInt RotatedSrcMask = MaskSrc.rotl(rotateAmt: SHMI);
3979 APInt FinalMask = RotatedSrcMask & MaskMI;
3980 uint32_t NewMB, NewME;
3981 bool Simplified = false;
3982
3983 // If final mask is 0, MI result should be 0 too.
3984 if (FinalMask.isZero()) {
3985 bool Is64Bit =
3986 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3987 Simplified = true;
3988 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3989 LLVM_DEBUG(MI.dump());
3990
3991 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3992 // Replace MI with "LI 0"
3993 MI.removeOperand(OpNo: 4);
3994 MI.removeOperand(OpNo: 3);
3995 MI.removeOperand(OpNo: 2);
3996 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: 0);
3997 MI.setDesc(get(Opcode: Is64Bit ? PPC::LI8 : PPC::LI));
3998 } else {
3999 // Replace MI with "ANDI_rec reg, 0"
4000 MI.removeOperand(OpNo: 4);
4001 MI.removeOperand(OpNo: 3);
4002 MI.getOperand(i: 2).setImm(0);
4003 MI.setDesc(get(Opcode: Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
4004 MI.getOperand(i: 1).setReg(SrcMI->getOperand(i: 1).getReg());
4005 if (SrcMI->getOperand(i: 1).isKill()) {
4006 MI.getOperand(i: 1).setIsKill(true);
4007 SrcMI->getOperand(i: 1).setIsKill(false);
4008 } else
4009 // About to replace MI.getOperand(1), clear its kill flag.
4010 MI.getOperand(i: 1).setIsKill(false);
4011 }
4012
4013 LLVM_DEBUG(dbgs() << "With: ");
4014 LLVM_DEBUG(MI.dump());
4015
4016 } else if ((isRunOfOnes(Val: (unsigned)(FinalMask.getZExtValue()), MB&: NewMB, ME&: NewME) &&
4017 NewMB <= NewME) ||
4018 SrcMaskFull) {
4019 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
4020 // than NewME. Otherwise we get a 64 bit value after folding, but MI
4021 // return a 32 bit value.
4022 Simplified = true;
4023 LLVM_DEBUG(dbgs() << "Converting Instr: ");
4024 LLVM_DEBUG(MI.dump());
4025
4026 uint16_t NewSH = (SHSrc + SHMI) % 32;
4027 MI.getOperand(i: 2).setImm(NewSH);
4028 // If SrcMI mask is full, no need to update MBMI and MEMI.
4029 if (!SrcMaskFull) {
4030 MI.getOperand(i: 3).setImm(NewMB);
4031 MI.getOperand(i: 4).setImm(NewME);
4032 }
4033 MI.getOperand(i: 1).setReg(SrcMI->getOperand(i: 1).getReg());
4034 if (SrcMI->getOperand(i: 1).isKill()) {
4035 MI.getOperand(i: 1).setIsKill(true);
4036 SrcMI->getOperand(i: 1).setIsKill(false);
4037 } else
4038 // About to replace MI.getOperand(1), clear its kill flag.
4039 MI.getOperand(i: 1).setIsKill(false);
4040
4041 LLVM_DEBUG(dbgs() << "To: ");
4042 LLVM_DEBUG(MI.dump());
4043 }
4044 if (Simplified & MRI->use_nodbg_empty(RegNo: FoldingReg) &&
4045 !SrcMI->hasImplicitDef()) {
4046 // If FoldingReg has no non-debug use and it has no implicit def (it
4047 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
4048 // Otherwise keep it.
4049 *ToErase = SrcMI;
4050 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
4051 LLVM_DEBUG(SrcMI->dump());
4052 }
4053 return Simplified;
4054}
4055
4056bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
4057 ImmInstrInfo &III, bool PostRA) const {
4058 // The vast majority of the instructions would need their operand 2 replaced
4059 // with an immediate when switching to the reg+imm form. A marked exception
4060 // are the update form loads/stores for which a constant operand 2 would need
4061 // to turn into a displacement and move operand 1 to the operand 2 position.
4062 III.ImmOpNo = 2;
4063 III.OpNoForForwarding = 2;
4064 III.ImmWidth = 16;
4065 III.ImmMustBeMultipleOf = 1;
4066 III.TruncateImmTo = 0;
4067 III.IsSummingOperands = false;
4068 switch (Opc) {
4069 default: return false;
4070 case PPC::ADD4:
4071 case PPC::ADD8:
4072 III.SignedImm = true;
4073 III.ZeroIsSpecialOrig = 0;
4074 III.ZeroIsSpecialNew = 1;
4075 III.IsCommutative = true;
4076 III.IsSummingOperands = true;
4077 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
4078 break;
4079 case PPC::ADDC:
4080 case PPC::ADDC8:
4081 III.SignedImm = true;
4082 III.ZeroIsSpecialOrig = 0;
4083 III.ZeroIsSpecialNew = 0;
4084 III.IsCommutative = true;
4085 III.IsSummingOperands = true;
4086 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
4087 break;
4088 case PPC::ADDC_rec:
4089 III.SignedImm = true;
4090 III.ZeroIsSpecialOrig = 0;
4091 III.ZeroIsSpecialNew = 0;
4092 III.IsCommutative = true;
4093 III.IsSummingOperands = true;
4094 III.ImmOpcode = PPC::ADDIC_rec;
4095 break;
4096 case PPC::SUBFC:
4097 case PPC::SUBFC8:
4098 III.SignedImm = true;
4099 III.ZeroIsSpecialOrig = 0;
4100 III.ZeroIsSpecialNew = 0;
4101 III.IsCommutative = false;
4102 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
4103 break;
4104 case PPC::CMPW:
4105 case PPC::CMPD:
4106 III.SignedImm = true;
4107 III.ZeroIsSpecialOrig = 0;
4108 III.ZeroIsSpecialNew = 0;
4109 III.IsCommutative = false;
4110 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
4111 break;
4112 case PPC::CMPLW:
4113 case PPC::CMPLD:
4114 III.SignedImm = false;
4115 III.ZeroIsSpecialOrig = 0;
4116 III.ZeroIsSpecialNew = 0;
4117 III.IsCommutative = false;
4118 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
4119 break;
4120 case PPC::AND_rec:
4121 case PPC::AND8_rec:
4122 case PPC::OR:
4123 case PPC::OR8:
4124 case PPC::XOR:
4125 case PPC::XOR8:
4126 III.SignedImm = false;
4127 III.ZeroIsSpecialOrig = 0;
4128 III.ZeroIsSpecialNew = 0;
4129 III.IsCommutative = true;
4130 switch(Opc) {
4131 default: llvm_unreachable("Unknown opcode");
4132 case PPC::AND_rec:
4133 III.ImmOpcode = PPC::ANDI_rec;
4134 break;
4135 case PPC::AND8_rec:
4136 III.ImmOpcode = PPC::ANDI8_rec;
4137 break;
4138 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
4139 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
4140 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
4141 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
4142 }
4143 break;
4144 case PPC::RLWNM:
4145 case PPC::RLWNM8:
4146 case PPC::RLWNM_rec:
4147 case PPC::RLWNM8_rec:
4148 case PPC::SLW:
4149 case PPC::SLW8:
4150 case PPC::SLW_rec:
4151 case PPC::SLW8_rec:
4152 case PPC::SRW:
4153 case PPC::SRW8:
4154 case PPC::SRW_rec:
4155 case PPC::SRW8_rec:
4156 case PPC::SRAW:
4157 case PPC::SRAW_rec:
4158 III.SignedImm = false;
4159 III.ZeroIsSpecialOrig = 0;
4160 III.ZeroIsSpecialNew = 0;
4161 III.IsCommutative = false;
4162 // This isn't actually true, but the instructions ignore any of the
4163 // upper bits, so any immediate loaded with an LI is acceptable.
4164 // This does not apply to shift right algebraic because a value
4165 // out of range will produce a -1/0.
4166 III.ImmWidth = 16;
4167 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4168 Opc == PPC::RLWNM8_rec)
4169 III.TruncateImmTo = 5;
4170 else
4171 III.TruncateImmTo = 6;
4172 switch(Opc) {
4173 default: llvm_unreachable("Unknown opcode");
4174 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4175 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4176 case PPC::RLWNM_rec:
4177 III.ImmOpcode = PPC::RLWINM_rec;
4178 break;
4179 case PPC::RLWNM8_rec:
4180 III.ImmOpcode = PPC::RLWINM8_rec;
4181 break;
4182 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4183 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4184 case PPC::SLW_rec:
4185 III.ImmOpcode = PPC::RLWINM_rec;
4186 break;
4187 case PPC::SLW8_rec:
4188 III.ImmOpcode = PPC::RLWINM8_rec;
4189 break;
4190 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4191 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4192 case PPC::SRW_rec:
4193 III.ImmOpcode = PPC::RLWINM_rec;
4194 break;
4195 case PPC::SRW8_rec:
4196 III.ImmOpcode = PPC::RLWINM8_rec;
4197 break;
4198 case PPC::SRAW:
4199 III.ImmWidth = 5;
4200 III.TruncateImmTo = 0;
4201 III.ImmOpcode = PPC::SRAWI;
4202 break;
4203 case PPC::SRAW_rec:
4204 III.ImmWidth = 5;
4205 III.TruncateImmTo = 0;
4206 III.ImmOpcode = PPC::SRAWI_rec;
4207 break;
4208 }
4209 break;
4210 case PPC::RLDCL:
4211 case PPC::RLDCL_rec:
4212 case PPC::RLDCR:
4213 case PPC::RLDCR_rec:
4214 case PPC::SLD:
4215 case PPC::SLD_rec:
4216 case PPC::SRD:
4217 case PPC::SRD_rec:
4218 case PPC::SRAD:
4219 case PPC::SRAD_rec:
4220 III.SignedImm = false;
4221 III.ZeroIsSpecialOrig = 0;
4222 III.ZeroIsSpecialNew = 0;
4223 III.IsCommutative = false;
4224 // This isn't actually true, but the instructions ignore any of the
4225 // upper bits, so any immediate loaded with an LI is acceptable.
4226 // This does not apply to shift right algebraic because a value
4227 // out of range will produce a -1/0.
4228 III.ImmWidth = 16;
4229 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4230 Opc == PPC::RLDCR_rec)
4231 III.TruncateImmTo = 6;
4232 else
4233 III.TruncateImmTo = 7;
4234 switch(Opc) {
4235 default: llvm_unreachable("Unknown opcode");
4236 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4237 case PPC::RLDCL_rec:
4238 III.ImmOpcode = PPC::RLDICL_rec;
4239 break;
4240 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4241 case PPC::RLDCR_rec:
4242 III.ImmOpcode = PPC::RLDICR_rec;
4243 break;
4244 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4245 case PPC::SLD_rec:
4246 III.ImmOpcode = PPC::RLDICR_rec;
4247 break;
4248 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4249 case PPC::SRD_rec:
4250 III.ImmOpcode = PPC::RLDICL_rec;
4251 break;
4252 case PPC::SRAD:
4253 III.ImmWidth = 6;
4254 III.TruncateImmTo = 0;
4255 III.ImmOpcode = PPC::SRADI;
4256 break;
4257 case PPC::SRAD_rec:
4258 III.ImmWidth = 6;
4259 III.TruncateImmTo = 0;
4260 III.ImmOpcode = PPC::SRADI_rec;
4261 break;
4262 }
4263 break;
4264 // Loads and stores:
4265 case PPC::LBZX:
4266 case PPC::LBZX8:
4267 case PPC::LHZX:
4268 case PPC::LHZX8:
4269 case PPC::LHAX:
4270 case PPC::LHAX8:
4271 case PPC::LWZX:
4272 case PPC::LWZX8:
4273 case PPC::LWAX:
4274 case PPC::LDX:
4275 case PPC::LFSX:
4276 case PPC::LFDX:
4277 case PPC::STBX:
4278 case PPC::STBX8:
4279 case PPC::STHX:
4280 case PPC::STHX8:
4281 case PPC::STWX:
4282 case PPC::STWX8:
4283 case PPC::STDX:
4284 case PPC::STFSX:
4285 case PPC::STFDX:
4286 III.SignedImm = true;
4287 III.ZeroIsSpecialOrig = 1;
4288 III.ZeroIsSpecialNew = 2;
4289 III.IsCommutative = true;
4290 III.IsSummingOperands = true;
4291 III.ImmOpNo = 1;
4292 III.OpNoForForwarding = 2;
4293 switch(Opc) {
4294 default: llvm_unreachable("Unknown opcode");
4295 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4296 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4297 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4298 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4299 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4300 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4301 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4302 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4303 case PPC::LWAX:
4304 III.ImmOpcode = PPC::LWA;
4305 III.ImmMustBeMultipleOf = 4;
4306 break;
4307 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4308 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4309 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4310 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4311 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4312 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4313 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4314 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4315 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4316 case PPC::STDX:
4317 III.ImmOpcode = PPC::STD;
4318 III.ImmMustBeMultipleOf = 4;
4319 break;
4320 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4321 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4322 }
4323 break;
4324 case PPC::LBZUX:
4325 case PPC::LBZUX8:
4326 case PPC::LHZUX:
4327 case PPC::LHZUX8:
4328 case PPC::LHAUX:
4329 case PPC::LHAUX8:
4330 case PPC::LWZUX:
4331 case PPC::LWZUX8:
4332 case PPC::LDUX:
4333 case PPC::LFSUX:
4334 case PPC::LFDUX:
4335 case PPC::STBUX:
4336 case PPC::STBUX8:
4337 case PPC::STHUX:
4338 case PPC::STHUX8:
4339 case PPC::STWUX:
4340 case PPC::STWUX8:
4341 case PPC::STDUX:
4342 case PPC::STFSUX:
4343 case PPC::STFDUX:
4344 III.SignedImm = true;
4345 III.ZeroIsSpecialOrig = 2;
4346 III.ZeroIsSpecialNew = 3;
4347 III.IsCommutative = false;
4348 III.IsSummingOperands = true;
4349 III.ImmOpNo = 2;
4350 III.OpNoForForwarding = 3;
4351 switch(Opc) {
4352 default: llvm_unreachable("Unknown opcode");
4353 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4354 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4355 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4356 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4357 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4358 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4359 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4360 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4361 case PPC::LDUX:
4362 III.ImmOpcode = PPC::LDU;
4363 III.ImmMustBeMultipleOf = 4;
4364 break;
4365 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4366 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4367 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4368 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4369 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4370 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4371 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4372 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4373 case PPC::STDUX:
4374 III.ImmOpcode = PPC::STDU;
4375 III.ImmMustBeMultipleOf = 4;
4376 break;
4377 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4378 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4379 }
4380 break;
4381 // Power9 and up only. For some of these, the X-Form version has access to all
4382 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4383 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4384 // into or stored from is one of the VR registers.
4385 case PPC::LXVX:
4386 case PPC::LXSSPX:
4387 case PPC::LXSDX:
4388 case PPC::STXVX:
4389 case PPC::STXSSPX:
4390 case PPC::STXSDX:
4391 case PPC::XFLOADf32:
4392 case PPC::XFLOADf64:
4393 case PPC::XFSTOREf32:
4394 case PPC::XFSTOREf64:
4395 if (!Subtarget.hasP9Vector())
4396 return false;
4397 III.SignedImm = true;
4398 III.ZeroIsSpecialOrig = 1;
4399 III.ZeroIsSpecialNew = 2;
4400 III.IsCommutative = true;
4401 III.IsSummingOperands = true;
4402 III.ImmOpNo = 1;
4403 III.OpNoForForwarding = 2;
4404 III.ImmMustBeMultipleOf = 4;
4405 switch(Opc) {
4406 default: llvm_unreachable("Unknown opcode");
4407 case PPC::LXVX:
4408 III.ImmOpcode = PPC::LXV;
4409 III.ImmMustBeMultipleOf = 16;
4410 break;
4411 case PPC::LXSSPX:
4412 if (PostRA) {
4413 if (IsVFReg)
4414 III.ImmOpcode = PPC::LXSSP;
4415 else {
4416 III.ImmOpcode = PPC::LFS;
4417 III.ImmMustBeMultipleOf = 1;
4418 }
4419 break;
4420 }
4421 [[fallthrough]];
4422 case PPC::XFLOADf32:
4423 III.ImmOpcode = PPC::DFLOADf32;
4424 break;
4425 case PPC::LXSDX:
4426 if (PostRA) {
4427 if (IsVFReg)
4428 III.ImmOpcode = PPC::LXSD;
4429 else {
4430 III.ImmOpcode = PPC::LFD;
4431 III.ImmMustBeMultipleOf = 1;
4432 }
4433 break;
4434 }
4435 [[fallthrough]];
4436 case PPC::XFLOADf64:
4437 III.ImmOpcode = PPC::DFLOADf64;
4438 break;
4439 case PPC::STXVX:
4440 III.ImmOpcode = PPC::STXV;
4441 III.ImmMustBeMultipleOf = 16;
4442 break;
4443 case PPC::STXSSPX:
4444 if (PostRA) {
4445 if (IsVFReg)
4446 III.ImmOpcode = PPC::STXSSP;
4447 else {
4448 III.ImmOpcode = PPC::STFS;
4449 III.ImmMustBeMultipleOf = 1;
4450 }
4451 break;
4452 }
4453 [[fallthrough]];
4454 case PPC::XFSTOREf32:
4455 III.ImmOpcode = PPC::DFSTOREf32;
4456 break;
4457 case PPC::STXSDX:
4458 if (PostRA) {
4459 if (IsVFReg)
4460 III.ImmOpcode = PPC::STXSD;
4461 else {
4462 III.ImmOpcode = PPC::STFD;
4463 III.ImmMustBeMultipleOf = 1;
4464 }
4465 break;
4466 }
4467 [[fallthrough]];
4468 case PPC::XFSTOREf64:
4469 III.ImmOpcode = PPC::DFSTOREf64;
4470 break;
4471 }
4472 break;
4473 }
4474 return true;
4475}
4476
4477// Utility function for swaping two arbitrary operands of an instruction.
4478static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4479 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4480
4481 unsigned MaxOp = std::max(a: Op1, b: Op2);
4482 unsigned MinOp = std::min(a: Op1, b: Op2);
4483 MachineOperand MOp1 = MI.getOperand(i: MinOp);
4484 MachineOperand MOp2 = MI.getOperand(i: MaxOp);
4485 MI.removeOperand(OpNo: std::max(a: Op1, b: Op2));
4486 MI.removeOperand(OpNo: std::min(a: Op1, b: Op2));
4487
4488 // If the operands we are swapping are the two at the end (the common case)
4489 // we can just remove both and add them in the opposite order.
4490 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4491 MI.addOperand(Op: MOp2);
4492 MI.addOperand(Op: MOp1);
4493 } else {
4494 // Store all operands in a temporary vector, remove them and re-add in the
4495 // right order.
4496 SmallVector<MachineOperand, 2> MOps;
4497 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4498 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4499 MOps.push_back(Elt: MI.getOperand(i));
4500 MI.removeOperand(OpNo: i);
4501 }
4502 // MOp2 needs to be added next.
4503 MI.addOperand(Op: MOp2);
4504 // Now add the rest.
4505 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4506 if (i == MaxOp)
4507 MI.addOperand(Op: MOp1);
4508 else {
4509 MI.addOperand(Op: MOps.back());
4510 MOps.pop_back();
4511 }
4512 }
4513 }
4514}
4515
4516// Check if the 'MI' that has the index OpNoForForwarding
4517// meets the requirement described in the ImmInstrInfo.
4518bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4519 const ImmInstrInfo &III,
4520 unsigned OpNoForForwarding
4521 ) const {
4522 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4523 // would not work pre-RA, we can only do the check post RA.
4524 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4525 if (MRI.isSSA())
4526 return false;
4527
4528 // Cannot do the transform if MI isn't summing the operands.
4529 if (!III.IsSummingOperands)
4530 return false;
4531
4532 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4533 if (!III.ZeroIsSpecialOrig)
4534 return false;
4535
4536 // We cannot do the transform if the operand we are trying to replace
4537 // isn't the same as the operand the instruction allows.
4538 if (OpNoForForwarding != III.OpNoForForwarding)
4539 return false;
4540
4541 // Check if the instruction we are trying to transform really has
4542 // the special zero register as its operand.
4543 if (MI.getOperand(i: III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4544 MI.getOperand(i: III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4545 return false;
4546
4547 // This machine instruction is convertible if it is,
4548 // 1. summing the operands.
4549 // 2. one of the operands is special zero register.
4550 // 3. the operand we are trying to replace is allowed by the MI.
4551 return true;
4552}
4553
4554// Check if the DefMI is the add inst and set the ImmMO and RegMO
4555// accordingly.
4556bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4557 const ImmInstrInfo &III,
4558 MachineOperand *&ImmMO,
4559 MachineOperand *&RegMO) const {
4560 unsigned Opc = DefMI.getOpcode();
4561 if (Opc != PPC::ADDItocL8 && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4562 return false;
4563
4564 // Skip the optimization of transformTo[NewImm|Imm]FormFedByAdd for ADDItocL8
4565 // on AIX which is used for toc-data access. TODO: Follow up to see if it can
4566 // apply for AIX toc-data as well.
4567 if (Opc == PPC::ADDItocL8 && Subtarget.isAIX())
4568 return false;
4569
4570 assert(DefMI.getNumOperands() >= 3 &&
4571 "Add inst must have at least three operands");
4572 RegMO = &DefMI.getOperand(i: 1);
4573 ImmMO = &DefMI.getOperand(i: 2);
4574
4575 // Before RA, ADDI first operand could be a frame index.
4576 if (!RegMO->isReg())
4577 return false;
4578
4579 // This DefMI is elgible for forwarding if it is:
4580 // 1. add inst
4581 // 2. one of the operands is Imm/CPI/Global.
4582 return isAnImmediateOperand(MO: *ImmMO);
4583}
4584
4585bool PPCInstrInfo::isRegElgibleForForwarding(
4586 const MachineOperand &RegMO, const MachineInstr &DefMI,
4587 const MachineInstr &MI, bool KillDefMI,
4588 bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4589 // x = addi y, imm
4590 // ...
4591 // z = lfdx 0, x -> z = lfd imm(y)
4592 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4593 // of "y" between the DEF of "x" and "z".
4594 // The query is only valid post RA.
4595 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4596 if (MRI.isSSA())
4597 return false;
4598
4599 Register Reg = RegMO.getReg();
4600
4601 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4602 MachineBasicBlock::const_reverse_iterator It = MI;
4603 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4604 It++;
4605 for (; It != E; ++It) {
4606 if (It->modifiesRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4607 return false;
4608 else if (It->killsRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4609 IsFwdFeederRegKilled = true;
4610 if (It->readsRegister(Reg, TRI: &getRegisterInfo()) && (&*It) != &DefMI)
4611 SeenIntermediateUse = true;
4612 // Made it to DefMI without encountering a clobber.
4613 if ((&*It) == &DefMI)
4614 break;
4615 }
4616 assert((&*It) == &DefMI && "DefMI is missing");
4617
4618 // If DefMI also defines the register to be forwarded, we can only forward it
4619 // if DefMI is being erased.
4620 if (DefMI.modifiesRegister(Reg, TRI: &getRegisterInfo()))
4621 return KillDefMI;
4622
4623 return true;
4624}
4625
4626bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4627 const MachineInstr &DefMI,
4628 const ImmInstrInfo &III,
4629 int64_t &Imm,
4630 int64_t BaseImm) const {
4631 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4632 if (DefMI.getOpcode() == PPC::ADDItocL8) {
4633 // The operand for ADDItocL8 is CPI, which isn't imm at compiling time,
4634 // However, we know that, it is 16-bit width, and has the alignment of 4.
4635 // Check if the instruction met the requirement.
4636 if (III.ImmMustBeMultipleOf > 4 ||
4637 III.TruncateImmTo || III.ImmWidth != 16)
4638 return false;
4639
4640 // Going from XForm to DForm loads means that the displacement needs to be
4641 // not just an immediate but also a multiple of 4, or 16 depending on the
4642 // load. A DForm load cannot be represented if it is a multiple of say 2.
4643 // XForm loads do not have this restriction.
4644 if (ImmMO.isGlobal()) {
4645 const DataLayout &DL = ImmMO.getGlobal()->getDataLayout();
4646 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
4647 return false;
4648 }
4649
4650 return true;
4651 }
4652
4653 if (ImmMO.isImm()) {
4654 // It is Imm, we need to check if the Imm fit the range.
4655 // Sign-extend to 64-bits.
4656 // DefMI may be folded with another imm form instruction, the result Imm is
4657 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4658 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4659 if (III.SignedImm && !ActualValue.isSignedIntN(N: III.ImmWidth))
4660 return false;
4661 if (!III.SignedImm && !ActualValue.isIntN(N: III.ImmWidth))
4662 return false;
4663 Imm = SignExtend64<16>(x: ImmMO.getImm() + BaseImm);
4664
4665 if (Imm % III.ImmMustBeMultipleOf)
4666 return false;
4667 if (III.TruncateImmTo)
4668 Imm &= ((1 << III.TruncateImmTo) - 1);
4669 }
4670 else
4671 return false;
4672
4673 // This ImmMO is forwarded if it meets the requriement describle
4674 // in ImmInstrInfo
4675 return true;
4676}
4677
4678bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4679 unsigned OpNoForForwarding,
4680 MachineInstr **KilledDef,
4681 SmallSet<Register, 4> *RegsToUpdate) const {
4682 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4683 !DefMI.getOperand(i: 1).isImm())
4684 return false;
4685
4686 MachineFunction *MF = MI.getParent()->getParent();
4687 MachineRegisterInfo *MRI = &MF->getRegInfo();
4688 bool PostRA = !MRI->isSSA();
4689
4690 int64_t Immediate = DefMI.getOperand(i: 1).getImm();
4691 // Sign-extend to 64-bits.
4692 int64_t SExtImm = SignExtend64<16>(x: Immediate);
4693
4694 bool ReplaceWithLI = false;
4695 bool Is64BitLI = false;
4696 int64_t NewImm = 0;
4697 bool SetCR = false;
4698 unsigned Opc = MI.getOpcode();
4699 switch (Opc) {
4700 default:
4701 return false;
4702
4703 // FIXME: Any branches conditional on such a comparison can be made
4704 // unconditional. At this time, this happens too infrequently to be worth
4705 // the implementation effort, but if that ever changes, we could convert
4706 // such a pattern here.
4707 case PPC::CMPWI:
4708 case PPC::CMPLWI:
4709 case PPC::CMPDI:
4710 case PPC::CMPLDI: {
4711 // Doing this post-RA would require dataflow analysis to reliably find uses
4712 // of the CR register set by the compare.
4713 // No need to fixup killed/dead flag since this transformation is only valid
4714 // before RA.
4715 if (PostRA)
4716 return false;
4717 // If a compare-immediate is fed by an immediate and is itself an input of
4718 // an ISEL (the most common case) into a COPY of the correct register.
4719 bool Changed = false;
4720 Register DefReg = MI.getOperand(i: 0).getReg();
4721 int64_t Comparand = MI.getOperand(i: 2).getImm();
4722 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4723 ? (Comparand | 0xFFFFFFFFFFFF0000)
4724 : Comparand;
4725
4726 for (auto &CompareUseMI : MRI->use_instructions(Reg: DefReg)) {
4727 unsigned UseOpc = CompareUseMI.getOpcode();
4728 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4729 continue;
4730 unsigned CRSubReg = CompareUseMI.getOperand(i: 3).getSubReg();
4731 Register TrueReg = CompareUseMI.getOperand(i: 1).getReg();
4732 Register FalseReg = CompareUseMI.getOperand(i: 2).getReg();
4733 unsigned RegToCopy =
4734 selectReg(Imm1: SExtImm, Imm2: SExtComparand, CompareOpc: Opc, TrueReg, FalseReg, CRSubReg);
4735 if (RegToCopy == PPC::NoRegister)
4736 continue;
4737 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4738 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4739 CompareUseMI.setDesc(get(Opcode: UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4740 replaceInstrOperandWithImm(MI&: CompareUseMI, OpNo: 1, Imm: 0);
4741 CompareUseMI.removeOperand(OpNo: 3);
4742 CompareUseMI.removeOperand(OpNo: 2);
4743 continue;
4744 }
4745 LLVM_DEBUG(
4746 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4747 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4748 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4749 if (RegsToUpdate) {
4750 for (const MachineOperand &MO : CompareUseMI.operands())
4751 if (MO.isReg())
4752 RegsToUpdate->insert(V: MO.getReg());
4753 }
4754 // Convert to copy and remove unneeded operands.
4755 CompareUseMI.setDesc(get(Opcode: PPC::COPY));
4756 CompareUseMI.removeOperand(OpNo: 3);
4757 CompareUseMI.removeOperand(OpNo: RegToCopy == TrueReg ? 2 : 1);
4758 CmpIselsConverted++;
4759 Changed = true;
4760 LLVM_DEBUG(CompareUseMI.dump());
4761 }
4762 if (Changed)
4763 return true;
4764 // This may end up incremented multiple times since this function is called
4765 // during a fixed-point transformation, but it is only meant to indicate the
4766 // presence of this opportunity.
4767 MissedConvertibleImmediateInstrs++;
4768 return false;
4769 }
4770
4771 // Immediate forms - may simply be convertable to an LI.
4772 case PPC::ADDI:
4773 case PPC::ADDI8: {
4774 // Does the sum fit in a 16-bit signed field?
4775 int64_t Addend = MI.getOperand(i: 2).getImm();
4776 if (isInt<16>(x: Addend + SExtImm)) {
4777 ReplaceWithLI = true;
4778 Is64BitLI = Opc == PPC::ADDI8;
4779 NewImm = Addend + SExtImm;
4780 break;
4781 }
4782 return false;
4783 }
4784 case PPC::SUBFIC:
4785 case PPC::SUBFIC8: {
4786 // Only transform this if the CARRY implicit operand is dead.
4787 if (MI.getNumOperands() > 3 && !MI.getOperand(i: 3).isDead())
4788 return false;
4789 int64_t Minuend = MI.getOperand(i: 2).getImm();
4790 if (isInt<16>(x: Minuend - SExtImm)) {
4791 ReplaceWithLI = true;
4792 Is64BitLI = Opc == PPC::SUBFIC8;
4793 NewImm = Minuend - SExtImm;
4794 break;
4795 }
4796 return false;
4797 }
4798 case PPC::RLDICL:
4799 case PPC::RLDICL_rec:
4800 case PPC::RLDICL_32:
4801 case PPC::RLDICL_32_64: {
4802 // Use APInt's rotate function.
4803 int64_t SH = MI.getOperand(i: 2).getImm();
4804 int64_t MB = MI.getOperand(i: 3).getImm();
4805 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4806 SExtImm, true);
4807 InVal = InVal.rotl(rotateAmt: SH);
4808 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4809 InVal &= Mask;
4810 // Can't replace negative values with an LI as that will sign-extend
4811 // and not clear the left bits. If we're setting the CR bit, we will use
4812 // ANDI_rec which won't sign extend, so that's safe.
4813 if (isUInt<15>(x: InVal.getSExtValue()) ||
4814 (Opc == PPC::RLDICL_rec && isUInt<16>(x: InVal.getSExtValue()))) {
4815 ReplaceWithLI = true;
4816 Is64BitLI = Opc != PPC::RLDICL_32;
4817 NewImm = InVal.getSExtValue();
4818 SetCR = Opc == PPC::RLDICL_rec;
4819 break;
4820 }
4821 return false;
4822 }
4823 case PPC::RLWINM:
4824 case PPC::RLWINM8:
4825 case PPC::RLWINM_rec:
4826 case PPC::RLWINM8_rec: {
4827 int64_t SH = MI.getOperand(i: 2).getImm();
4828 int64_t MB = MI.getOperand(i: 3).getImm();
4829 int64_t ME = MI.getOperand(i: 4).getImm();
4830 APInt InVal(32, SExtImm, true);
4831 InVal = InVal.rotl(rotateAmt: SH);
4832 APInt Mask = APInt::getBitsSetWithWrap(numBits: 32, loBit: 32 - ME - 1, hiBit: 32 - MB);
4833 InVal &= Mask;
4834 // Can't replace negative values with an LI as that will sign-extend
4835 // and not clear the left bits. If we're setting the CR bit, we will use
4836 // ANDI_rec which won't sign extend, so that's safe.
4837 bool ValueFits = isUInt<15>(x: InVal.getSExtValue());
4838 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4839 isUInt<16>(x: InVal.getSExtValue()));
4840 if (ValueFits) {
4841 ReplaceWithLI = true;
4842 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4843 NewImm = InVal.getSExtValue();
4844 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4845 break;
4846 }
4847 return false;
4848 }
4849 case PPC::ORI:
4850 case PPC::ORI8:
4851 case PPC::XORI:
4852 case PPC::XORI8: {
4853 int64_t LogicalImm = MI.getOperand(i: 2).getImm();
4854 int64_t Result = 0;
4855 if (Opc == PPC::ORI || Opc == PPC::ORI8)
4856 Result = LogicalImm | SExtImm;
4857 else
4858 Result = LogicalImm ^ SExtImm;
4859 if (isInt<16>(x: Result)) {
4860 ReplaceWithLI = true;
4861 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4862 NewImm = Result;
4863 break;
4864 }
4865 return false;
4866 }
4867 }
4868
4869 if (ReplaceWithLI) {
4870 // We need to be careful with CR-setting instructions we're replacing.
4871 if (SetCR) {
4872 // We don't know anything about uses when we're out of SSA, so only
4873 // replace if the new immediate will be reproduced.
4874 bool ImmChanged = (SExtImm & NewImm) != NewImm;
4875 if (PostRA && ImmChanged)
4876 return false;
4877
4878 if (!PostRA) {
4879 // If the defining load-immediate has no other uses, we can just replace
4880 // the immediate with the new immediate.
4881 if (MRI->hasOneUse(RegNo: DefMI.getOperand(i: 0).getReg()))
4882 DefMI.getOperand(i: 1).setImm(NewImm);
4883
4884 // If we're not using the GPR result of the CR-setting instruction, we
4885 // just need to and with zero/non-zero depending on the new immediate.
4886 else if (MRI->use_empty(RegNo: MI.getOperand(i: 0).getReg())) {
4887 if (NewImm) {
4888 assert(Immediate && "Transformation converted zero to non-zero?");
4889 NewImm = Immediate;
4890 }
4891 } else if (ImmChanged)
4892 return false;
4893 }
4894 }
4895
4896 LLVM_DEBUG(dbgs() << "Replacing constant instruction:\n");
4897 LLVM_DEBUG(MI.dump());
4898 LLVM_DEBUG(dbgs() << "Fed by:\n");
4899 LLVM_DEBUG(DefMI.dump());
4900 LoadImmediateInfo LII;
4901 LII.Imm = NewImm;
4902 LII.Is64Bit = Is64BitLI;
4903 LII.SetCR = SetCR;
4904 // If we're setting the CR, the original load-immediate must be kept (as an
4905 // operand to ANDI_rec/ANDI8_rec).
4906 if (KilledDef && SetCR)
4907 *KilledDef = nullptr;
4908 replaceInstrWithLI(MI, LII);
4909
4910 if (PostRA)
4911 recomputeLivenessFlags(MBB&: *MI.getParent());
4912
4913 LLVM_DEBUG(dbgs() << "With:\n");
4914 LLVM_DEBUG(MI.dump());
4915 return true;
4916 }
4917 return false;
4918}
4919
4920bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4921 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4922 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4923 bool PostRA = !MRI->isSSA();
4924 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4925 // for post-ra.
4926 if (PostRA)
4927 return false;
4928
4929 // Only handle load/store.
4930 if (!MI.mayLoadOrStore())
4931 return false;
4932
4933 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(ImmOpcode: MI.getOpcode());
4934
4935 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4936 "MI must have x-form opcode");
4937
4938 // get Imm Form info.
4939 ImmInstrInfo III;
4940 bool IsVFReg = MI.getOperand(i: 0).isReg() &&
4941 MI.getOperand(i: 0).getReg().isPhysical() &&
4942 PPC::isVFRegister(Reg: MI.getOperand(i: 0).getReg());
4943
4944 if (!instrHasImmForm(Opc: XFormOpcode, IsVFReg, III, PostRA))
4945 return false;
4946
4947 if (!III.IsSummingOperands)
4948 return false;
4949
4950 if (OpNoForForwarding != III.OpNoForForwarding)
4951 return false;
4952
4953 MachineOperand ImmOperandMI = MI.getOperand(i: III.ImmOpNo);
4954 if (!ImmOperandMI.isImm())
4955 return false;
4956
4957 // Check DefMI.
4958 MachineOperand *ImmMO = nullptr;
4959 MachineOperand *RegMO = nullptr;
4960 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4961 return false;
4962 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4963
4964 // Check Imm.
4965 // Set ImmBase from imm instruction as base and get new Imm inside
4966 // isImmElgibleForForwarding.
4967 int64_t ImmBase = ImmOperandMI.getImm();
4968 int64_t Imm = 0;
4969 if (!isImmElgibleForForwarding(ImmMO: *ImmMO, DefMI, III, Imm, BaseImm: ImmBase))
4970 return false;
4971
4972 // Do the transform
4973 LLVM_DEBUG(dbgs() << "Replacing existing reg+imm instruction:\n");
4974 LLVM_DEBUG(MI.dump());
4975 LLVM_DEBUG(dbgs() << "Fed by:\n");
4976 LLVM_DEBUG(DefMI.dump());
4977
4978 MI.getOperand(i: III.OpNoForForwarding).setReg(RegMO->getReg());
4979 MI.getOperand(i: III.ImmOpNo).setImm(Imm);
4980
4981 LLVM_DEBUG(dbgs() << "With:\n");
4982 LLVM_DEBUG(MI.dump());
4983 return true;
4984}
4985
4986// If an X-Form instruction is fed by an add-immediate and one of its operands
4987// is the literal zero, attempt to forward the source of the add-immediate to
4988// the corresponding D-Form instruction with the displacement coming from
4989// the immediate being added.
4990bool PPCInstrInfo::transformToImmFormFedByAdd(
4991 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4992 MachineInstr &DefMI, bool KillDefMI) const {
4993 // RegMO ImmMO
4994 // | |
4995 // x = addi reg, imm <----- DefMI
4996 // y = op 0 , x <----- MI
4997 // |
4998 // OpNoForForwarding
4999 // Check if the MI meet the requirement described in the III.
5000 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
5001 return false;
5002
5003 // Check if the DefMI meet the requirement
5004 // described in the III. If yes, set the ImmMO and RegMO accordingly.
5005 MachineOperand *ImmMO = nullptr;
5006 MachineOperand *RegMO = nullptr;
5007 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
5008 return false;
5009 assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
5010
5011 // As we get the Imm operand now, we need to check if the ImmMO meet
5012 // the requirement described in the III. If yes set the Imm.
5013 int64_t Imm = 0;
5014 if (!isImmElgibleForForwarding(ImmMO: *ImmMO, DefMI, III, Imm))
5015 return false;
5016
5017 bool IsFwdFeederRegKilled = false;
5018 bool SeenIntermediateUse = false;
5019 // Check if the RegMO can be forwarded to MI.
5020 if (!isRegElgibleForForwarding(RegMO: *RegMO, DefMI, MI, KillDefMI,
5021 IsFwdFeederRegKilled, SeenIntermediateUse))
5022 return false;
5023
5024 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5025 bool PostRA = !MRI.isSSA();
5026
5027 // We know that, the MI and DefMI both meet the pattern, and
5028 // the Imm also meet the requirement with the new Imm-form.
5029 // It is safe to do the transformation now.
5030 LLVM_DEBUG(dbgs() << "Replacing indexed instruction:\n");
5031 LLVM_DEBUG(MI.dump());
5032 LLVM_DEBUG(dbgs() << "Fed by:\n");
5033 LLVM_DEBUG(DefMI.dump());
5034
5035 // Update the base reg first.
5036 MI.getOperand(i: III.OpNoForForwarding).ChangeToRegister(Reg: RegMO->getReg(),
5037 isDef: false, isImp: false,
5038 isKill: RegMO->isKill());
5039
5040 // Then, update the imm.
5041 if (ImmMO->isImm()) {
5042 // If the ImmMO is Imm, change the operand that has ZERO to that Imm
5043 // directly.
5044 replaceInstrOperandWithImm(MI, OpNo: III.ZeroIsSpecialOrig, Imm);
5045 }
5046 else {
5047 // Otherwise, it is Constant Pool Index(CPI) or Global,
5048 // which is relocation in fact. We need to replace the special zero
5049 // register with ImmMO.
5050 // Before that, we need to fixup the target flags for imm.
5051 // For some reason, we miss to set the flag for the ImmMO if it is CPI.
5052 if (DefMI.getOpcode() == PPC::ADDItocL8)
5053 ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
5054
5055 // MI didn't have the interface such as MI.setOperand(i) though
5056 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
5057 // ImmMO, we need to remove ZERO operand and all the operands behind it,
5058 // and, add the ImmMO, then, move back all the operands behind ZERO.
5059 SmallVector<MachineOperand, 2> MOps;
5060 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
5061 MOps.push_back(Elt: MI.getOperand(i));
5062 MI.removeOperand(OpNo: i);
5063 }
5064
5065 // Remove the last MO in the list, which is ZERO operand in fact.
5066 MOps.pop_back();
5067 // Add the imm operand.
5068 MI.addOperand(Op: *ImmMO);
5069 // Now add the rest back.
5070 for (auto &MO : MOps)
5071 MI.addOperand(Op: MO);
5072 }
5073
5074 // Update the opcode.
5075 MI.setDesc(get(Opcode: III.ImmOpcode));
5076
5077 if (PostRA)
5078 recomputeLivenessFlags(MBB&: *MI.getParent());
5079 LLVM_DEBUG(dbgs() << "With:\n");
5080 LLVM_DEBUG(MI.dump());
5081
5082 return true;
5083}
5084
5085bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
5086 const ImmInstrInfo &III,
5087 unsigned ConstantOpNo,
5088 MachineInstr &DefMI) const {
5089 // DefMI must be LI or LI8.
5090 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
5091 !DefMI.getOperand(i: 1).isImm())
5092 return false;
5093
5094 // Get Imm operand and Sign-extend to 64-bits.
5095 int64_t Imm = SignExtend64<16>(x: DefMI.getOperand(i: 1).getImm());
5096
5097 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5098 bool PostRA = !MRI.isSSA();
5099 // Exit early if we can't convert this.
5100 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
5101 return false;
5102 if (Imm % III.ImmMustBeMultipleOf)
5103 return false;
5104 if (III.TruncateImmTo)
5105 Imm &= ((1 << III.TruncateImmTo) - 1);
5106 if (III.SignedImm) {
5107 APInt ActualValue(64, Imm, true);
5108 if (!ActualValue.isSignedIntN(N: III.ImmWidth))
5109 return false;
5110 } else {
5111 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
5112 if ((uint64_t)Imm > UnsignedMax)
5113 return false;
5114 }
5115
5116 // If we're post-RA, the instructions don't agree on whether register zero is
5117 // special, we can transform this as long as the register operand that will
5118 // end up in the location where zero is special isn't R0.
5119 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5120 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
5121 III.ZeroIsSpecialNew + 1;
5122 Register OrigZeroReg = MI.getOperand(i: PosForOrigZero).getReg();
5123 Register NewZeroReg = MI.getOperand(i: III.ZeroIsSpecialNew).getReg();
5124 // If R0 is in the operand where zero is special for the new instruction,
5125 // it is unsafe to transform if the constant operand isn't that operand.
5126 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
5127 ConstantOpNo != III.ZeroIsSpecialNew)
5128 return false;
5129 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
5130 ConstantOpNo != PosForOrigZero)
5131 return false;
5132 }
5133
5134 unsigned Opc = MI.getOpcode();
5135 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
5136 Opc == PPC::SRW || Opc == PPC::SRW_rec ||
5137 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
5138 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
5139 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
5140 Opc == PPC::SRD || Opc == PPC::SRD_rec;
5141 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
5142 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
5143 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
5144 Opc == PPC::SRD_rec;
5145
5146 LLVM_DEBUG(dbgs() << "Replacing reg+reg instruction: ");
5147 LLVM_DEBUG(MI.dump());
5148 LLVM_DEBUG(dbgs() << "Fed by load-immediate: ");
5149 LLVM_DEBUG(DefMI.dump());
5150 MI.setDesc(get(Opcode: III.ImmOpcode));
5151 if (ConstantOpNo == III.OpNoForForwarding) {
5152 // Converting shifts to immediate form is a bit tricky since they may do
5153 // one of three things:
5154 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
5155 // 2. If the shift amount is zero, the result is unchanged (save for maybe
5156 // setting CR0)
5157 // 3. If the shift amount is in [1, OpSize), it's just a shift
5158 if (SpecialShift32 || SpecialShift64) {
5159 LoadImmediateInfo LII;
5160 LII.Imm = 0;
5161 LII.SetCR = SetCR;
5162 LII.Is64Bit = SpecialShift64;
5163 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
5164 if (Imm & (SpecialShift32 ? 0x20 : 0x40))
5165 replaceInstrWithLI(MI, LII);
5166 // Shifts by zero don't change the value. If we don't need to set CR0,
5167 // just convert this to a COPY. Can't do this post-RA since we've already
5168 // cleaned up the copies.
5169 else if (!SetCR && ShAmt == 0 && !PostRA) {
5170 MI.removeOperand(OpNo: 2);
5171 MI.setDesc(get(Opcode: PPC::COPY));
5172 } else {
5173 // The 32 bit and 64 bit instructions are quite different.
5174 if (SpecialShift32) {
5175 // Left shifts use (N, 0, 31-N).
5176 // Right shifts use (32-N, N, 31) if 0 < N < 32.
5177 // use (0, 0, 31) if N == 0.
5178 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
5179 uint64_t MB = RightShift ? ShAmt : 0;
5180 uint64_t ME = RightShift ? 31 : 31 - ShAmt;
5181 replaceInstrOperandWithImm(MI, OpNo: III.OpNoForForwarding, Imm: SH);
5182 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(Val: MB)
5183 .addImm(Val: ME);
5184 } else {
5185 // Left shifts use (N, 63-N).
5186 // Right shifts use (64-N, N) if 0 < N < 64.
5187 // use (0, 0) if N == 0.
5188 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
5189 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
5190 replaceInstrOperandWithImm(MI, OpNo: III.OpNoForForwarding, Imm: SH);
5191 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(Val: ME);
5192 }
5193 }
5194 } else
5195 replaceInstrOperandWithImm(MI, OpNo: ConstantOpNo, Imm);
5196 }
5197 // Convert commutative instructions (switch the operands and convert the
5198 // desired one to an immediate.
5199 else if (III.IsCommutative) {
5200 replaceInstrOperandWithImm(MI, OpNo: ConstantOpNo, Imm);
5201 swapMIOperands(MI, Op1: ConstantOpNo, Op2: III.OpNoForForwarding);
5202 } else
5203 llvm_unreachable("Should have exited early!");
5204
5205 // For instructions for which the constant register replaces a different
5206 // operand than where the immediate goes, we need to swap them.
5207 if (III.OpNoForForwarding != III.ImmOpNo)
5208 swapMIOperands(MI, Op1: III.OpNoForForwarding, Op2: III.ImmOpNo);
5209
5210 // If the special R0/X0 register index are different for original instruction
5211 // and new instruction, we need to fix up the register class in new
5212 // instruction.
5213 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5214 if (III.ZeroIsSpecialNew) {
5215 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
5216 // need to fix up register class.
5217 Register RegToModify = MI.getOperand(i: III.ZeroIsSpecialNew).getReg();
5218 if (RegToModify.isVirtual()) {
5219 const TargetRegisterClass *NewRC =
5220 MRI.getRegClass(Reg: RegToModify)->hasSuperClassEq(RC: &PPC::GPRCRegClass) ?
5221 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
5222 MRI.setRegClass(Reg: RegToModify, RC: NewRC);
5223 }
5224 }
5225 }
5226
5227 if (PostRA)
5228 recomputeLivenessFlags(MBB&: *MI.getParent());
5229
5230 LLVM_DEBUG(dbgs() << "With: ");
5231 LLVM_DEBUG(MI.dump());
5232 LLVM_DEBUG(dbgs() << "\n");
5233 return true;
5234}
5235
5236const TargetRegisterClass *
5237PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
5238 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
5239 return &PPC::VSRCRegClass;
5240 return RC;
5241}
5242
5243int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
5244 return PPC::getRecordFormOpcode(Opcode);
5245}
5246
5247static bool isOpZeroOfSubwordPreincLoad(int Opcode) {
5248 return (Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 ||
5249 Opcode == PPC::LBZUX8 || Opcode == PPC::LHZU ||
5250 Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 ||
5251 Opcode == PPC::LHZUX8);
5252}
5253
5254// This function checks for sign extension from 32 bits to 64 bits.
5255static bool definedBySignExtendingOp(const PPCInstrInfo &TII,
5256 const unsigned Reg,
5257 const MachineRegisterInfo *MRI) {
5258 if (!Register::isVirtualRegister(Reg))
5259 return false;
5260
5261 MachineInstr *MI = MRI->getVRegDef(Reg);
5262 if (!MI)
5263 return false;
5264
5265 int Opcode = MI->getOpcode();
5266 if (TII.isSExt32To64(Opcode))
5267 return true;
5268
5269 // The first def of LBZU/LHZU is sign extended.
5270 if (isOpZeroOfSubwordPreincLoad(Opcode) && MI->getOperand(i: 0).getReg() == Reg)
5271 return true;
5272
5273 // RLDICL generates sign-extended output if it clears at least
5274 // 33 bits from the left (MSB).
5275 if (Opcode == PPC::RLDICL && MI->getOperand(i: 3).getImm() >= 33)
5276 return true;
5277
5278 // If at least one bit from left in a lower word is masked out,
5279 // all of 0 to 32-th bits of the output are cleared.
5280 // Hence the output is already sign extended.
5281 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5282 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
5283 MI->getOperand(i: 3).getImm() > 0 &&
5284 MI->getOperand(i: 3).getImm() <= MI->getOperand(i: 4).getImm())
5285 return true;
5286
5287 // If the most significant bit of immediate in ANDIS is zero,
5288 // all of 0 to 32-th bits are cleared.
5289 if (Opcode == PPC::ANDIS_rec || Opcode == PPC::ANDIS8_rec) {
5290 uint16_t Imm = MI->getOperand(i: 2).getImm();
5291 if ((Imm & 0x8000) == 0)
5292 return true;
5293 }
5294
5295 return false;
5296}
5297
5298// This function checks the machine instruction that defines the input register
5299// Reg. If that machine instruction always outputs a value that has only zeros
5300// in the higher 32 bits then this function will return true.
5301static bool definedByZeroExtendingOp(const PPCInstrInfo &TII,
5302 const unsigned Reg,
5303 const MachineRegisterInfo *MRI) {
5304 if (!Register::isVirtualRegister(Reg))
5305 return false;
5306
5307 MachineInstr *MI = MRI->getVRegDef(Reg);
5308 if (!MI)
5309 return false;
5310
5311 int Opcode = MI->getOpcode();
5312 if (TII.isZExt32To64(Opcode))
5313 return true;
5314
5315 // The first def of LBZU/LHZU/LWZU are zero extended.
5316 if ((isOpZeroOfSubwordPreincLoad(Opcode) || Opcode == PPC::LWZU ||
5317 Opcode == PPC::LWZUX || Opcode == PPC::LWZU8 || Opcode == PPC::LWZUX8) &&
5318 MI->getOperand(i: 0).getReg() == Reg)
5319 return true;
5320
5321 // The 16-bit immediate is sign-extended in li/lis.
5322 // If the most significant bit is zero, all higher bits are zero.
5323 if (Opcode == PPC::LI || Opcode == PPC::LI8 ||
5324 Opcode == PPC::LIS || Opcode == PPC::LIS8) {
5325 int64_t Imm = MI->getOperand(i: 1).getImm();
5326 if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
5327 return true;
5328 }
5329
5330 // We have some variations of rotate-and-mask instructions
5331 // that clear higher 32-bits.
5332 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
5333 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
5334 Opcode == PPC::RLDICL_32_64) &&
5335 MI->getOperand(i: 3).getImm() >= 32)
5336 return true;
5337
5338 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
5339 MI->getOperand(i: 3).getImm() >= 32 &&
5340 MI->getOperand(i: 3).getImm() <= 63 - MI->getOperand(i: 2).getImm())
5341 return true;
5342
5343 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5344 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
5345 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
5346 MI->getOperand(i: 3).getImm() <= MI->getOperand(i: 4).getImm())
5347 return true;
5348
5349 return false;
5350}
5351
5352// This function returns true if the input MachineInstr is a TOC save
5353// instruction.
5354bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
5355 if (!MI.getOperand(i: 1).isImm() || !MI.getOperand(i: 2).isReg())
5356 return false;
5357 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5358 unsigned StackOffset = MI.getOperand(i: 1).getImm();
5359 Register StackReg = MI.getOperand(i: 2).getReg();
5360 Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
5361 if (StackReg == SPReg && StackOffset == TOCSaveOffset)
5362 return true;
5363
5364 return false;
5365}
5366
5367// We limit the max depth to track incoming values of PHIs or binary ops
5368// (e.g. AND) to avoid excessive cost.
5369const unsigned MAX_BINOP_DEPTH = 1;
5370
5371// This function will promote the instruction which defines the register `Reg`
5372// in the parameter from a 32-bit to a 64-bit instruction if needed. The logic
5373// used to check whether an instruction needs to be promoted or not is similar
5374// to the logic used to check whether or not a defined register is sign or zero
5375// extended within the function PPCInstrInfo::isSignOrZeroExtended.
5376// Additionally, the `promoteInstr32To64ForElimEXTSW` function is recursive.
5377// BinOpDepth does not count all of the recursions. The parameter BinOpDepth is
5378// incremented only when `promoteInstr32To64ForElimEXTSW` calls itself more
5379// than once. This is done to prevent exponential recursion.
5380void PPCInstrInfo::promoteInstr32To64ForElimEXTSW(const Register &Reg,
5381 MachineRegisterInfo *MRI,
5382 unsigned BinOpDepth) const {
5383 if (!Reg.isVirtual())
5384 return;
5385
5386 MachineInstr *MI = MRI->getVRegDef(Reg);
5387 if (!MI)
5388 return;
5389
5390 unsigned Opcode = MI->getOpcode();
5391
5392 switch (Opcode) {
5393 case PPC::OR:
5394 case PPC::ISEL:
5395 case PPC::OR8:
5396 case PPC::PHI: {
5397 if (BinOpDepth >= MAX_BINOP_DEPTH)
5398 break;
5399 unsigned OperandEnd = 3, OperandStride = 1;
5400 if (Opcode == PPC::PHI) {
5401 OperandEnd = MI->getNumOperands();
5402 OperandStride = 2;
5403 }
5404
5405 for (unsigned I = 1; I < OperandEnd; I += OperandStride) {
5406 assert(MI->getOperand(I).isReg() && "Operand must be register");
5407 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: I).getReg(), MRI,
5408 BinOpDepth: BinOpDepth + 1);
5409 }
5410
5411 break;
5412 }
5413 case PPC::COPY: {
5414 // Refers to the logic of the `case PPC::COPY` statement in the function
5415 // PPCInstrInfo::isSignOrZeroExtended().
5416
5417 Register SrcReg = MI->getOperand(i: 1).getReg();
5418 // In both ELFv1 and v2 ABI, method parameters and the return value
5419 // are sign- or zero-extended.
5420 const MachineFunction *MF = MI->getMF();
5421 if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5422 // If this is a copy from another register, we recursively promote the
5423 // source.
5424 promoteInstr32To64ForElimEXTSW(Reg: SrcReg, MRI, BinOpDepth);
5425 return;
5426 }
5427
5428 // From here on everything is SVR4ABI. COPY will be eliminated in the other
5429 // pass, we do not need promote the COPY pseudo opcode.
5430
5431 if (SrcReg != PPC::X3)
5432 // If this is a copy from another register, we recursively promote the
5433 // source.
5434 promoteInstr32To64ForElimEXTSW(Reg: SrcReg, MRI, BinOpDepth);
5435 return;
5436 }
5437 case PPC::ORI:
5438 case PPC::XORI:
5439 case PPC::ORIS:
5440 case PPC::XORIS:
5441 case PPC::ORI8:
5442 case PPC::XORI8:
5443 case PPC::ORIS8:
5444 case PPC::XORIS8:
5445 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 1).getReg(), MRI, BinOpDepth);
5446 break;
5447 case PPC::AND:
5448 case PPC::AND8:
5449 if (BinOpDepth >= MAX_BINOP_DEPTH)
5450 break;
5451
5452 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 1).getReg(), MRI,
5453 BinOpDepth: BinOpDepth + 1);
5454 promoteInstr32To64ForElimEXTSW(Reg: MI->getOperand(i: 2).getReg(), MRI,
5455 BinOpDepth: BinOpDepth + 1);
5456 break;
5457 }
5458
5459 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
5460 if (RC == &PPC::G8RCRegClass || RC == &PPC::G8RC_and_G8RC_NOX0RegClass)
5461 return;
5462
5463 // Map the 32bit to 64bit opcodes for instructions that are not signed or zero
5464 // extended themselves, but may have operands who's destination registers of
5465 // signed or zero extended instructions.
5466 DenseMap<unsigned, unsigned> OpcodeMap = {
5467 {PPC::OR, PPC::OR8}, {PPC::ISEL, PPC::ISEL8},
5468 {PPC::ORI, PPC::ORI8}, {PPC::XORI, PPC::XORI8},
5469 {PPC::ORIS, PPC::ORIS8}, {PPC::XORIS, PPC::XORIS8},
5470 {PPC::AND, PPC::AND8}};
5471
5472 int NewOpcode = -1;
5473 auto It = OpcodeMap.find(Val: Opcode);
5474 if (It != OpcodeMap.end()) {
5475 // Set the new opcode to the mapped 64-bit version.
5476 NewOpcode = It->second;
5477 } else {
5478 if (!isSExt32To64(Opcode))
5479 return;
5480
5481 // The TableGen function `get64BitInstrFromSignedExt32BitInstr` is used to
5482 // map the 32-bit instruction with the `SExt32To64` flag to the 64-bit
5483 // instruction with the same opcode.
5484 NewOpcode = PPC::get64BitInstrFromSignedExt32BitInstr(Opcode);
5485 }
5486
5487 assert(NewOpcode != -1 &&
5488 "Must have a 64-bit opcode to map the 32-bit opcode!");
5489
5490 const MCInstrDesc &MCID = get(Opcode: NewOpcode);
5491 const TargetRegisterClass *NewRC =
5492 RI.getRegClass(i: MCID.operands()[0].RegClass);
5493
5494 Register SrcReg = MI->getOperand(i: 0).getReg();
5495 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
5496
5497 // If the register class of the defined register in the 32-bit instruction
5498 // is the same as the register class of the defined register in the promoted
5499 // 64-bit instruction, we do not need to promote the instruction.
5500 if (NewRC == SrcRC)
5501 return;
5502
5503 DebugLoc DL = MI->getDebugLoc();
5504 auto MBB = MI->getParent();
5505
5506 // Since the pseudo-opcode of the instruction is promoted from 32-bit to
5507 // 64-bit, if the source reg class of the original instruction belongs to
5508 // PPC::GRCRegClass or PPC::GPRC_and_GPRC_NOR0RegClass, we need to promote
5509 // the operand to PPC::G8CRegClass or PPC::G8RC_and_G8RC_NOR0RegClass,
5510 // respectively.
5511 for (unsigned i = 1; i < MI->getNumOperands(); i++) {
5512 MachineOperand &Operand = MI->getOperand(i);
5513 if (!Operand.isReg())
5514 continue;
5515
5516 Register OperandReg = Operand.getReg();
5517 if (!OperandReg.isVirtual())
5518 continue;
5519
5520 const TargetRegisterClass *NewUsedRegRC =
5521 RI.getRegClass(i: MCID.operands()[i].RegClass);
5522 const TargetRegisterClass *OrgRC = MRI->getRegClass(Reg: OperandReg);
5523
5524 // An operand that reads the sub_32 subregister of a 64-bit register already
5525 // holds the value in a full 64-bit register, so it can be used directly by
5526 // the promoted instruction.
5527 if (Operand.getSubReg() == PPC::sub_32) {
5528 assert(NewUsedRegRC->hasSubClassEq(OrgRC) &&
5529 "sub_32 source register class is incompatible with the promoted "
5530 "operand register class");
5531 Operand.setSubReg(PPC::NoSubRegister);
5532 continue;
5533 }
5534
5535 if (NewUsedRegRC != OrgRC && (OrgRC == &PPC::GPRCRegClass ||
5536 OrgRC == &PPC::GPRC_and_GPRC_NOR0RegClass)) {
5537 // Promote the used 32-bit register to 64-bit register.
5538 Register TmpReg = MRI->createVirtualRegister(RegClass: NewUsedRegRC);
5539 Register DstTmpReg = MRI->createVirtualRegister(RegClass: NewUsedRegRC);
5540 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: PPC::IMPLICIT_DEF), DestReg: TmpReg);
5541 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: PPC::INSERT_SUBREG), DestReg: DstTmpReg)
5542 .addReg(RegNo: TmpReg)
5543 .addReg(RegNo: OperandReg)
5544 .addImm(Val: PPC::sub_32);
5545 Operand.setReg(DstTmpReg);
5546 Operand.setIsKill();
5547 }
5548 }
5549
5550 Register NewDefinedReg = MRI->createVirtualRegister(RegClass: NewRC);
5551 MI->setDesc(MCID);
5552 MI->getOperand(i: 0).setReg(NewDefinedReg);
5553
5554 // A defined register may be used by other instructions that are 32-bit.
5555 // After the defined register is promoted to 64-bit for the promoted
5556 // instruction, we need to demote the 64-bit defined register back to a
5557 // 32-bit register
5558 BuildMI(BB&: *MBB, I: std::next(x: MI->getIterator()), MIMD: DL, MCID: get(Opcode: PPC::COPY), DestReg: SrcReg)
5559 .addReg(RegNo: NewDefinedReg, Flags: RegState::Kill, SubReg: PPC::sub_32);
5560}
5561
5562// The isSignOrZeroExtended function is recursive. The parameter BinOpDepth
5563// does not count all of the recursions. The parameter BinOpDepth is incremented
5564// only when isSignOrZeroExtended calls itself more than once. This is done to
5565// prevent expontential recursion. There is no parameter to track linear
5566// recursion.
5567std::pair<bool, bool>
5568PPCInstrInfo::isSignOrZeroExtended(const unsigned Reg,
5569 const unsigned BinOpDepth,
5570 const MachineRegisterInfo *MRI) const {
5571 if (!Register::isVirtualRegister(Reg))
5572 return std::pair<bool, bool>(false, false);
5573
5574 MachineInstr *MI = MRI->getVRegDef(Reg);
5575 if (!MI)
5576 return std::pair<bool, bool>(false, false);
5577
5578 bool IsSExt = definedBySignExtendingOp(TII: *this, Reg, MRI);
5579 bool IsZExt = definedByZeroExtendingOp(TII: *this, Reg, MRI);
5580
5581 // If we know the instruction always returns sign- and zero-extended result,
5582 // return here.
5583 if (IsSExt && IsZExt)
5584 return std::pair<bool, bool>(IsSExt, IsZExt);
5585
5586 switch (MI->getOpcode()) {
5587 case PPC::COPY: {
5588 Register SrcReg = MI->getOperand(i: 1).getReg();
5589
5590 // In both ELFv1 and v2 ABI, method parameters and the return value
5591 // are sign- or zero-extended.
5592 const MachineFunction *MF = MI->getMF();
5593
5594 if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5595 // If this is a copy from another register, we recursively check source.
5596 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5597 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5598 SrcExt.second || IsZExt);
5599 }
5600
5601 // From here on everything is SVR4ABI
5602 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
5603 // We check the ZExt/SExt flags for a method parameter.
5604 if (MI->getParent()->getBasicBlock() ==
5605 &MF->getFunction().getEntryBlock()) {
5606 Register VReg = MI->getOperand(i: 0).getReg();
5607 if (MF->getRegInfo().isLiveIn(Reg: VReg)) {
5608 IsSExt |= FuncInfo->isLiveInSExt(VReg);
5609 IsZExt |= FuncInfo->isLiveInZExt(VReg);
5610 return std::pair<bool, bool>(IsSExt, IsZExt);
5611 }
5612 }
5613
5614 if (SrcReg != PPC::X3) {
5615 // If this is a copy from another register, we recursively check source.
5616 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5617 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5618 SrcExt.second || IsZExt);
5619 }
5620
5621 // For a method return value, we check the ZExt/SExt flags in attribute.
5622 // We assume the following code sequence for method call.
5623 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
5624 // BL8_NOP @func,...
5625 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
5626 // %5 = COPY %x3; G8RC:%5
5627 const MachineBasicBlock *MBB = MI->getParent();
5628 std::pair<bool, bool> IsExtendPair = std::pair<bool, bool>(IsSExt, IsZExt);
5629 MachineBasicBlock::const_instr_iterator II =
5630 MachineBasicBlock::const_instr_iterator(MI);
5631 if (II == MBB->instr_begin() || (--II)->getOpcode() != PPC::ADJCALLSTACKUP)
5632 return IsExtendPair;
5633
5634 const MachineInstr &CallMI = *(--II);
5635 if (!CallMI.isCall() || !CallMI.getOperand(i: 0).isGlobal())
5636 return IsExtendPair;
5637
5638 const Function *CalleeFn =
5639 dyn_cast_if_present<Function>(Val: CallMI.getOperand(i: 0).getGlobal());
5640 if (!CalleeFn)
5641 return IsExtendPair;
5642 const IntegerType *IntTy = dyn_cast<IntegerType>(Val: CalleeFn->getReturnType());
5643 if (IntTy && IntTy->getBitWidth() <= 32) {
5644 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
5645 IsSExt |= Attrs.hasAttribute(Kind: Attribute::SExt);
5646 IsZExt |= Attrs.hasAttribute(Kind: Attribute::ZExt);
5647 return std::pair<bool, bool>(IsSExt, IsZExt);
5648 }
5649
5650 return IsExtendPair;
5651 }
5652
5653 // OR, XOR with 16-bit immediate does not change the upper 48 bits.
5654 // So, we track the operand register as we do for register copy.
5655 case PPC::ORI:
5656 case PPC::XORI:
5657 case PPC::ORI8:
5658 case PPC::XORI8: {
5659 Register SrcReg = MI->getOperand(i: 1).getReg();
5660 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5661 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5662 SrcExt.second || IsZExt);
5663 }
5664
5665 // OR, XOR with shifted 16-bit immediate does not change the upper
5666 // 32 bits. So, we track the operand register for zero extension.
5667 // For sign extension when the MSB of the immediate is zero, we also
5668 // track the operand register since the upper 33 bits are unchanged.
5669 case PPC::ORIS:
5670 case PPC::XORIS:
5671 case PPC::ORIS8:
5672 case PPC::XORIS8: {
5673 Register SrcReg = MI->getOperand(i: 1).getReg();
5674 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth, MRI);
5675 uint16_t Imm = MI->getOperand(i: 2).getImm();
5676 if (Imm & 0x8000)
5677 return std::pair<bool, bool>(false, SrcExt.second || IsZExt);
5678 else
5679 return std::pair<bool, bool>(SrcExt.first || IsSExt,
5680 SrcExt.second || IsZExt);
5681 }
5682
5683 // If all incoming values are sign-/zero-extended,
5684 // the output of OR, ISEL or PHI is also sign-/zero-extended.
5685 case PPC::OR:
5686 case PPC::OR8:
5687 case PPC::ISEL:
5688 case PPC::PHI: {
5689 if (BinOpDepth >= MAX_BINOP_DEPTH)
5690 return std::pair<bool, bool>(false, false);
5691
5692 // The input registers for PHI are operand 1, 3, ...
5693 // The input registers for others are operand 1 and 2.
5694 unsigned OperandEnd = 3, OperandStride = 1;
5695 if (MI->getOpcode() == PPC::PHI) {
5696 OperandEnd = MI->getNumOperands();
5697 OperandStride = 2;
5698 }
5699
5700 IsSExt = true;
5701 IsZExt = true;
5702 for (unsigned I = 1; I != OperandEnd; I += OperandStride) {
5703 if (!MI->getOperand(i: I).isReg())
5704 return std::pair<bool, bool>(false, false);
5705
5706 Register SrcReg = MI->getOperand(i: I).getReg();
5707 auto SrcExt = isSignOrZeroExtended(Reg: SrcReg, BinOpDepth: BinOpDepth + 1, MRI);
5708 IsSExt &= SrcExt.first;
5709 IsZExt &= SrcExt.second;
5710 }
5711 return std::pair<bool, bool>(IsSExt, IsZExt);
5712 }
5713
5714 // If at least one of the incoming values of an AND is zero extended
5715 // then the output is also zero-extended. If both of the incoming values
5716 // are sign-extended then the output is also sign extended.
5717 case PPC::AND:
5718 case PPC::AND8: {
5719 if (BinOpDepth >= MAX_BINOP_DEPTH)
5720 return std::pair<bool, bool>(false, false);
5721
5722 Register SrcReg1 = MI->getOperand(i: 1).getReg();
5723 Register SrcReg2 = MI->getOperand(i: 2).getReg();
5724 auto Src1Ext = isSignOrZeroExtended(Reg: SrcReg1, BinOpDepth: BinOpDepth + 1, MRI);
5725 auto Src2Ext = isSignOrZeroExtended(Reg: SrcReg2, BinOpDepth: BinOpDepth + 1, MRI);
5726 return std::pair<bool, bool>(Src1Ext.first && Src2Ext.first,
5727 Src1Ext.second || Src2Ext.second);
5728 }
5729
5730 default:
5731 break;
5732 }
5733 return std::pair<bool, bool>(IsSExt, IsZExt);
5734}
5735
5736bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
5737 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
5738}
5739
5740namespace {
5741class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5742 MachineInstr *Loop, *EndLoop, *LoopCount;
5743 MachineFunction *MF;
5744 const TargetInstrInfo *TII;
5745 int64_t TripCount;
5746
5747public:
5748 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
5749 MachineInstr *LoopCount)
5750 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
5751 MF(Loop->getParent()->getParent()),
5752 TII(MF->getSubtarget().getInstrInfo()) {
5753 // Inspect the Loop instruction up-front, as it may be deleted when we call
5754 // createTripCountGreaterCondition.
5755 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
5756 TripCount = LoopCount->getOperand(i: 1).getImm();
5757 else
5758 TripCount = -1;
5759 }
5760
5761 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5762 // Only ignore the terminator.
5763 return MI == EndLoop;
5764 }
5765
5766 std::optional<bool> createTripCountGreaterCondition(
5767 int TC, MachineBasicBlock &MBB,
5768 SmallVectorImpl<MachineOperand> &Cond) override {
5769 if (TripCount == -1) {
5770 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5771 // so we don't need to generate any thing here.
5772 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
5773 Cond.push_back(Elt: MachineOperand::CreateReg(
5774 Reg: MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
5775 isDef: true));
5776 return {};
5777 }
5778
5779 return TripCount > TC;
5780 }
5781
5782 void setPreheader(MachineBasicBlock *NewPreheader) override {
5783 // Do nothing. We want the LOOP setup instruction to stay in the *old*
5784 // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
5785 }
5786
5787 void adjustTripCount(int TripCountAdjust) override {
5788 // If the loop trip count is a compile-time value, then just change the
5789 // value.
5790 if (LoopCount->getOpcode() == PPC::LI8 ||
5791 LoopCount->getOpcode() == PPC::LI) {
5792 int64_t TripCount = LoopCount->getOperand(i: 1).getImm() + TripCountAdjust;
5793 LoopCount->getOperand(i: 1).setImm(TripCount);
5794 return;
5795 }
5796
5797 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5798 // so we don't need to generate any thing here.
5799 }
5800
5801 void disposed(LiveIntervals *LIS) override {
5802 if (LIS) {
5803 LIS->RemoveMachineInstrFromMaps(MI&: *Loop);
5804 LIS->RemoveMachineInstrFromMaps(MI&: *LoopCount);
5805 }
5806 Loop->eraseFromParent();
5807 // Ensure the loop setup instruction is deleted too.
5808 LoopCount->eraseFromParent();
5809 }
5810};
5811} // namespace
5812
5813std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5814PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5815 // We really "analyze" only hardware loops right now.
5816 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
5817 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
5818 if (Preheader == LoopBB)
5819 Preheader = *std::next(x: LoopBB->pred_begin());
5820 MachineFunction *MF = Preheader->getParent();
5821
5822 if (I != LoopBB->end() && isBDNZ(Opcode: I->getOpcode())) {
5823 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5824 if (MachineInstr *LoopInst = findLoopInstr(PreHeader&: *Preheader, Visited)) {
5825 Register LoopCountReg = LoopInst->getOperand(i: 0).getReg();
5826 MachineRegisterInfo &MRI = MF->getRegInfo();
5827 MachineInstr *LoopCount = MRI.getUniqueVRegDef(Reg: LoopCountReg);
5828 return std::make_unique<PPCPipelinerLoopInfo>(args&: LoopInst, args: &*I, args&: LoopCount);
5829 }
5830 }
5831 return nullptr;
5832}
5833
5834MachineInstr *PPCInstrInfo::findLoopInstr(
5835 MachineBasicBlock &PreHeader,
5836 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
5837
5838 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
5839
5840 // The loop set-up instruction should be in preheader
5841 for (auto &I : PreHeader.instrs())
5842 if (I.getOpcode() == LOOPi)
5843 return &I;
5844 return nullptr;
5845}
5846
5847// Return true if get the base operand, byte offset of an instruction and the
5848// memory width. Width is the size of memory that is being loaded/stored.
5849bool PPCInstrInfo::getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
5850 const MachineOperand *&BaseReg,
5851 int64_t &Offset,
5852 LocationSize &Width) const {
5853 if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
5854 return false;
5855
5856 // Handle only loads/stores with base register followed by immediate offset.
5857 if (!LdSt.getOperand(i: 1).isImm() ||
5858 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()))
5859 return false;
5860
5861 if (!LdSt.hasOneMemOperand())
5862 return false;
5863
5864 Width = (*LdSt.memoperands_begin())->getSize();
5865 Offset = LdSt.getOperand(i: 1).getImm();
5866 BaseReg = &LdSt.getOperand(i: 2);
5867 return true;
5868}
5869
5870bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
5871 const MachineInstr &MIa, const MachineInstr &MIb) const {
5872 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
5873 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
5874
5875 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
5876 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
5877 return false;
5878
5879 // Retrieve the base register, offset from the base register and width. Width
5880 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
5881 // base registers are identical, and the offset of a lower memory access +
5882 // the width doesn't overlap the offset of a higher memory access,
5883 // then the memory accesses are different.
5884 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
5885 int64_t OffsetA = 0, OffsetB = 0;
5886 LocationSize WidthA = LocationSize::precise(Value: 0),
5887 WidthB = LocationSize::precise(Value: 0);
5888 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA) &&
5889 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB)) {
5890 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
5891 int LowOffset = std::min(a: OffsetA, b: OffsetB);
5892 int HighOffset = std::max(a: OffsetA, b: OffsetB);
5893 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
5894 if (LowWidth.hasValue() &&
5895 LowOffset + (int)LowWidth.getValue() <= HighOffset)
5896 return true;
5897 }
5898 }
5899 return false;
5900}
5901
5902// Expands LWAT_CSNE_PSEUDO/LDAT_CSNE_PSEUDO post register allocation.
5903// lwat/ldat FC=16 requires 3 consecutive registers. X8/X9/X10 are
5904// hardcoded post-RA to satisfy this constraint without a dedicated
5905// register class.
5906bool PPCInstrInfo::expandAMOCSNEPseudo(MachineInstr &MI) const {
5907 MachineBasicBlock &MBB = *MI.getParent();
5908 DebugLoc DL = MI.getDebugLoc();
5909 bool IsLDAT = MI.getOpcode() == PPC::LDAT_CSNE_PSEUDO;
5910
5911 Register DstReg = MI.getOperand(i: 0).getReg();
5912 Register PtrReg = MI.getOperand(i: 1).getReg();
5913
5914 Register ScratchReg = PtrReg;
5915 if (PtrReg == PPC::X8 || PtrReg == PPC::X9 || PtrReg == PPC::X10) {
5916 // If ptr is in X8/X9/X10, use $dst as scratch to move ptr away from
5917 // X8/X9/X10 since lwat FC=16 always writes its result to X8. After lwat
5918 // copy X8 into $dst.
5919 Register DstReg64 = IsLDAT ? DstReg
5920 : Register(RI.getMatchingSuperReg(
5921 Reg: DstReg, SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass));
5922 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: PPC::OR8), DestReg: DstReg64).addReg(RegNo: PtrReg).addReg(RegNo: PtrReg);
5923 ScratchReg = DstReg64;
5924 }
5925
5926 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: IsLDAT ? PPC::LDAT_CSNE : PPC::LWAT_CSNE), DestReg: PPC::X8)
5927 .addReg(RegNo: ScratchReg)
5928 .addReg(RegNo: PPC::X9, Flags: RegState::Implicit)
5929 .addReg(RegNo: PPC::X10, Flags: RegState::Implicit);
5930
5931 if (DstReg != (IsLDAT ? PPC::X8 : PPC::R8)) {
5932 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: IsLDAT ? PPC::OR8 : PPC::OR), DestReg: DstReg)
5933 .addReg(RegNo: IsLDAT ? PPC::X8 : PPC::R8)
5934 .addReg(RegNo: IsLDAT ? PPC::X8 : PPC::R8);
5935 }
5936 MI.eraseFromParent();
5937 return true;
5938}
5939