1//===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the Base ARM implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ARMBaseInstrInfo.h"
14#include "ARMBaseRegisterInfo.h"
15#include "ARMConstantPoolValue.h"
16#include "ARMFeatures.h"
17#include "ARMHazardRecognizer.h"
18#include "ARMMachineFunctionInfo.h"
19#include "ARMSubtarget.h"
20#include "MCTargetDesc/ARMAddressingModes.h"
21#include "MCTargetDesc/ARMBaseInfo.h"
22#include "MVETailPredUtils.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/DFAPacketizer.h"
29#include "llvm/CodeGen/LiveVariables.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineConstantPool.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachinePipeliner.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/MachineScheduler.h"
42#include "llvm/CodeGen/MultiHazardRecognizer.h"
43#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/TargetInstrInfo.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/CodeGen/TargetSchedule.h"
48#include "llvm/IR/Attributes.h"
49#include "llvm/IR/DebugLoc.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/GlobalValue.h"
52#include "llvm/IR/Module.h"
53#include "llvm/MC/MCAsmInfo.h"
54#include "llvm/MC/MCInstrDesc.h"
55#include "llvm/MC/MCInstrItineraries.h"
56#include "llvm/Support/BranchProbability.h"
57#include "llvm/Support/Casting.h"
58#include "llvm/Support/Compiler.h"
59#include "llvm/Support/Debug.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/raw_ostream.h"
62#include "llvm/Target/TargetMachine.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <iterator>
67#include <new>
68#include <utility>
69#include <vector>
70
71using namespace llvm;
72
73#define DEBUG_TYPE "arm-instrinfo"
74
75#define GET_INSTRINFO_CTOR_DTOR
76#include "ARMGenInstrInfo.inc"
77
78/// ARM_MLxEntry - Record information about MLA / MLS instructions.
79struct ARM_MLxEntry {
80 uint16_t MLxOpc; // MLA / MLS opcode
81 uint16_t MulOpc; // Expanded multiplication opcode
82 uint16_t AddSubOpc; // Expanded add / sub opcode
83 bool NegAcc; // True if the acc is negated before the add / sub.
84 bool HasLane; // True if instruction has an extra "lane" operand.
85};
86
87static const ARM_MLxEntry ARM_MLxTable[] = {
88 // MLxOpc, MulOpc, AddSubOpc, NegAcc, HasLane
89 // fp scalar ops
90 { .MLxOpc: ARM::VMLAS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VADDS, .NegAcc: false, .HasLane: false },
91 { .MLxOpc: ARM::VMLSS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: false, .HasLane: false },
92 { .MLxOpc: ARM::VMLAD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VADDD, .NegAcc: false, .HasLane: false },
93 { .MLxOpc: ARM::VMLSD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: false, .HasLane: false },
94 { .MLxOpc: ARM::VNMLAS, .MulOpc: ARM::VNMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: true, .HasLane: false },
95 { .MLxOpc: ARM::VNMLSS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: true, .HasLane: false },
96 { .MLxOpc: ARM::VNMLAD, .MulOpc: ARM::VNMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: true, .HasLane: false },
97 { .MLxOpc: ARM::VNMLSD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: true, .HasLane: false },
98
99 // fp SIMD ops
100 { .MLxOpc: ARM::VMLAfd, .MulOpc: ARM::VMULfd, .AddSubOpc: ARM::VADDfd, .NegAcc: false, .HasLane: false },
101 { .MLxOpc: ARM::VMLSfd, .MulOpc: ARM::VMULfd, .AddSubOpc: ARM::VSUBfd, .NegAcc: false, .HasLane: false },
102 { .MLxOpc: ARM::VMLAfq, .MulOpc: ARM::VMULfq, .AddSubOpc: ARM::VADDfq, .NegAcc: false, .HasLane: false },
103 { .MLxOpc: ARM::VMLSfq, .MulOpc: ARM::VMULfq, .AddSubOpc: ARM::VSUBfq, .NegAcc: false, .HasLane: false },
104 { .MLxOpc: ARM::VMLAslfd, .MulOpc: ARM::VMULslfd, .AddSubOpc: ARM::VADDfd, .NegAcc: false, .HasLane: true },
105 { .MLxOpc: ARM::VMLSslfd, .MulOpc: ARM::VMULslfd, .AddSubOpc: ARM::VSUBfd, .NegAcc: false, .HasLane: true },
106 { .MLxOpc: ARM::VMLAslfq, .MulOpc: ARM::VMULslfq, .AddSubOpc: ARM::VADDfq, .NegAcc: false, .HasLane: true },
107 { .MLxOpc: ARM::VMLSslfq, .MulOpc: ARM::VMULslfq, .AddSubOpc: ARM::VSUBfq, .NegAcc: false, .HasLane: true },
108};
109
110ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget &STI,
111 const ARMBaseRegisterInfo &TRI)
112 : ARMGenInstrInfo(STI, TRI, ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
113 Subtarget(STI) {
114 for (unsigned i = 0, e = std::size(ARM_MLxTable); i != e; ++i) {
115 if (!MLxEntryMap.insert(KV: std::make_pair(x: ARM_MLxTable[i].MLxOpc, y&: i)).second)
116 llvm_unreachable("Duplicated entries?");
117 MLxHazardOpcodes.insert(V: ARM_MLxTable[i].AddSubOpc);
118 MLxHazardOpcodes.insert(V: ARM_MLxTable[i].MulOpc);
119 }
120}
121
122// Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
123// currently defaults to no prepass hazard recognizer.
124ScheduleHazardRecognizer *
125ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
126 const ScheduleDAG *DAG) const {
127 if (usePreRAHazardRecognizer()) {
128 const InstrItineraryData *II =
129 static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
130 return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
131 }
132 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
133}
134
135// Called during:
136// - pre-RA scheduling
137// - post-RA scheduling when FeatureUseMISched is set
138ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer(
139 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
140 MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
141
142 // We would like to restrict this hazard recognizer to only
143 // post-RA scheduling; we can tell that we're post-RA because we don't
144 // track VRegLiveness.
145 // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM
146 // banks banked on bit 2. Assume that TCMs are in use.
147 if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness())
148 MHR->AddHazardRecognizer(
149 std::make_unique<ARMBankConflictHazardRecognizer>(args&: DAG, args: 0x4, args: true));
150
151 // Not inserting ARMHazardRecognizerFPMLx because that would change
152 // legacy behavior
153
154 auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
155 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
156 return MHR;
157}
158
159// Called during post-RA scheduling when FeatureUseMISched is not set
160ScheduleHazardRecognizer *ARMBaseInstrInfo::
161CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
162 const ScheduleDAG *DAG) const {
163 MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
164
165 if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
166 MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>());
167
168 auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
169 if (BHR)
170 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
171 return MHR;
172}
173
174// Branch analysis.
175// Cond vector output format:
176// 0 elements indicates an unconditional branch
177// 2 elements indicates a conditional branch; the elements are
178// the condition to check and the CPSR.
179// 3 elements indicates a hardware loop end; the elements
180// are the opcode, the operand value to test, and a dummy
181// operand used to pad out to 3 operands.
182bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
183 MachineBasicBlock *&TBB,
184 MachineBasicBlock *&FBB,
185 SmallVectorImpl<MachineOperand> &Cond,
186 bool AllowModify) const {
187 TBB = nullptr;
188 FBB = nullptr;
189
190 MachineBasicBlock::instr_iterator I = MBB.instr_end();
191 if (I == MBB.instr_begin())
192 return false; // Empty blocks are easy.
193 --I;
194
195 // Walk backwards from the end of the basic block until the branch is
196 // analyzed or we give up.
197 while (isPredicated(MI: *I) || I->isTerminator() || I->isDebugValue()) {
198 // Flag to be raised on unanalyzeable instructions. This is useful in cases
199 // where we want to clean up on the end of the basic block before we bail
200 // out.
201 bool CantAnalyze = false;
202
203 // Skip over DEBUG values, predicated nonterminators and speculation
204 // barrier terminators.
205 while (I->isDebugInstr() || !I->isTerminator() ||
206 isSpeculationBarrierEndBBOpcode(Opc: I->getOpcode()) ||
207 I->getOpcode() == ARM::t2DoLoopStartTP){
208 if (I == MBB.instr_begin())
209 return false;
210 --I;
211 }
212
213 if (isIndirectBranchOpcode(Opc: I->getOpcode()) ||
214 isJumpTableBranchOpcode(Opc: I->getOpcode())) {
215 // Indirect branches and jump tables can't be analyzed, but we still want
216 // to clean up any instructions at the tail of the basic block.
217 CantAnalyze = true;
218 } else if (isUncondBranchOpcode(Opc: I->getOpcode())) {
219 TBB = I->getOperand(i: 0).getMBB();
220 } else if (isCondBranchOpcode(Opc: I->getOpcode())) {
221 // Bail out if we encounter multiple conditional branches.
222 if (!Cond.empty())
223 return true;
224
225 assert(!FBB && "FBB should have been null.");
226 FBB = TBB;
227 TBB = I->getOperand(i: 0).getMBB();
228 Cond.push_back(Elt: I->getOperand(i: 1));
229 Cond.push_back(Elt: I->getOperand(i: 2));
230 } else if (I->isReturn()) {
231 // Returns can't be analyzed, but we should run cleanup.
232 CantAnalyze = true;
233 } else if (I->getOpcode() == ARM::t2LoopEnd &&
234 MBB.getParent()
235 ->getSubtarget<ARMSubtarget>()
236 .enableMachinePipeliner()) {
237 if (!Cond.empty())
238 return true;
239 FBB = TBB;
240 TBB = I->getOperand(i: 1).getMBB();
241 Cond.push_back(Elt: MachineOperand::CreateImm(Val: I->getOpcode()));
242 Cond.push_back(Elt: I->getOperand(i: 0));
243 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
244 } else {
245 // We encountered other unrecognized terminator. Bail out immediately.
246 return true;
247 }
248
249 // Cleanup code - to be run for unpredicated unconditional branches and
250 // returns.
251 if (!isPredicated(MI: *I) &&
252 (isUncondBranchOpcode(Opc: I->getOpcode()) ||
253 isIndirectBranchOpcode(Opc: I->getOpcode()) ||
254 isJumpTableBranchOpcode(Opc: I->getOpcode()) ||
255 I->isReturn())) {
256 // Forget any previous condition branch information - it no longer applies.
257 Cond.clear();
258 FBB = nullptr;
259
260 // If we can modify the function, delete everything below this
261 // unconditional branch.
262 if (AllowModify) {
263 MachineBasicBlock::iterator DI = std::next(x: I);
264 while (DI != MBB.instr_end()) {
265 MachineInstr &InstToDelete = *DI;
266 ++DI;
267 // Speculation barriers must not be deleted.
268 if (isSpeculationBarrierEndBBOpcode(Opc: InstToDelete.getOpcode()))
269 continue;
270 InstToDelete.eraseFromParent();
271 }
272 }
273 }
274
275 if (CantAnalyze) {
276 // We may not be able to analyze the block, but we could still have
277 // an unconditional branch as the last instruction in the block, which
278 // just branches to layout successor. If this is the case, then just
279 // remove it if we're allowed to make modifications.
280 if (AllowModify && !isPredicated(MI: MBB.back()) &&
281 isUncondBranchOpcode(Opc: MBB.back().getOpcode()) &&
282 TBB && MBB.isLayoutSuccessor(MBB: TBB))
283 removeBranch(MBB);
284 return true;
285 }
286
287 if (I == MBB.instr_begin())
288 return false;
289
290 --I;
291 }
292
293 // We made it past the terminators without bailing out - we must have
294 // analyzed this branch successfully.
295 return false;
296}
297
298unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
299 int *BytesRemoved) const {
300 assert(!BytesRemoved && "code size not handled");
301
302 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
303 if (I == MBB.end())
304 return 0;
305
306 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
307 !isCondBranchOpcode(Opc: I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
308 return 0;
309
310 // Remove the branch.
311 I->eraseFromParent();
312
313 I = MBB.end();
314
315 if (I == MBB.begin()) return 1;
316 --I;
317 if (!isCondBranchOpcode(Opc: I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
318 return 1;
319
320 // Remove the branch.
321 I->eraseFromParent();
322 return 2;
323}
324
325unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
326 MachineBasicBlock *TBB,
327 MachineBasicBlock *FBB,
328 ArrayRef<MachineOperand> Cond,
329 const DebugLoc &DL,
330 int *BytesAdded) const {
331 assert(!BytesAdded && "code size not handled");
332 ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
333 int BOpc = !AFI->isThumbFunction()
334 ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
335 int BccOpc = !AFI->isThumbFunction()
336 ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
337 bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
338
339 // Shouldn't be a fall through.
340 assert(TBB && "insertBranch must not be told to insert a fallthrough");
341 assert((Cond.size() == 2 || Cond.size() == 0 || Cond.size() == 3) &&
342 "ARM branch conditions have two or three components!");
343
344 // For conditional branches, we use addOperand to preserve CPSR flags.
345
346 if (!FBB) {
347 if (Cond.empty()) { // Unconditional branch?
348 if (isThumb)
349 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: TBB).add(MOs: predOps(Pred: ARMCC::AL));
350 else
351 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: TBB);
352 } else if (Cond.size() == 2) {
353 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BccOpc))
354 .addMBB(MBB: TBB)
355 .addImm(Val: Cond[0].getImm())
356 .add(MO: Cond[1]);
357 } else
358 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm())).add(MO: Cond[1]).addMBB(MBB: TBB);
359 return 1;
360 }
361
362 // Two-way conditional branch.
363 if (Cond.size() == 2)
364 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BccOpc))
365 .addMBB(MBB: TBB)
366 .addImm(Val: Cond[0].getImm())
367 .add(MO: Cond[1]);
368 else if (Cond.size() == 3)
369 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm())).add(MO: Cond[1]).addMBB(MBB: TBB);
370 if (isThumb)
371 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: FBB).add(MOs: predOps(Pred: ARMCC::AL));
372 else
373 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: FBB);
374 return 2;
375}
376
377bool ARMBaseInstrInfo::
378reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
379 if (Cond.size() == 2) {
380 ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
381 Cond[0].setImm(ARMCC::getOppositeCondition(CC));
382 return false;
383 }
384 return true;
385}
386
387bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
388 if (MI.isBundle()) {
389 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
390 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
391 while (++I != E && I->isInsideBundle()) {
392 int PIdx = I->findFirstPredOperandIdx();
393 if (PIdx != -1 && I->getOperand(i: PIdx).getImm() != ARMCC::AL)
394 return true;
395 }
396 return false;
397 }
398
399 int PIdx = MI.findFirstPredOperandIdx();
400 return PIdx != -1 && MI.getOperand(i: PIdx).getImm() != ARMCC::AL;
401}
402
403std::string ARMBaseInstrInfo::createMIROperandComment(
404 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
405 const TargetRegisterInfo *TRI) const {
406
407 // First, let's see if there is a generic comment for this operand
408 std::string GenericComment =
409 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
410 if (!GenericComment.empty())
411 return GenericComment;
412
413 // If not, check if we have an immediate operand.
414 if (!Op.isImm())
415 return std::string();
416
417 // And print its corresponding condition code if the immediate is a
418 // predicate.
419 int FirstPredOp = MI.findFirstPredOperandIdx();
420 if (FirstPredOp != (int) OpIdx)
421 return std::string();
422
423 std::string CC = "CC::";
424 CC += ARMCondCodeToString(CC: (ARMCC::CondCodes)Op.getImm());
425 return CC;
426}
427
428bool ARMBaseInstrInfo::PredicateInstruction(
429 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
430 unsigned Opc = MI.getOpcode();
431 if (isUncondBranchOpcode(Opc)) {
432 MI.setDesc(get(Opcode: getMatchingCondBranchOpcode(Opc)));
433 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
434 .addImm(Val: Pred[0].getImm())
435 .addReg(RegNo: Pred[1].getReg());
436 return true;
437 }
438
439 int PIdx = MI.findFirstPredOperandIdx();
440 if (PIdx != -1) {
441 MachineOperand &PMO = MI.getOperand(i: PIdx);
442 PMO.setImm(Pred[0].getImm());
443 MI.getOperand(i: PIdx+1).setReg(Pred[1].getReg());
444
445 // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
446 // IT block. This affects how they are printed.
447 const MCInstrDesc &MCID = MI.getDesc();
448 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
449 assert(MCID.operands()[1].isOptionalDef() &&
450 "CPSR def isn't expected operand");
451 assert((MI.getOperand(1).isDead() ||
452 MI.getOperand(1).getReg() != ARM::CPSR) &&
453 "if conversion tried to stop defining used CPSR");
454 MI.getOperand(i: 1).setReg(ARM::NoRegister);
455 }
456
457 return true;
458 }
459 return false;
460}
461
462bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
463 ArrayRef<MachineOperand> Pred2) const {
464 if (Pred1.size() > 2 || Pred2.size() > 2)
465 return false;
466
467 ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
468 ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
469 if (CC1 == CC2)
470 return true;
471
472 switch (CC1) {
473 default:
474 return false;
475 case ARMCC::AL:
476 return true;
477 case ARMCC::HS:
478 return CC2 == ARMCC::HI;
479 case ARMCC::LS:
480 return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
481 case ARMCC::GE:
482 return CC2 == ARMCC::GT;
483 case ARMCC::LE:
484 return CC2 == ARMCC::LT;
485 }
486}
487
488bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI,
489 std::vector<MachineOperand> &Pred,
490 bool SkipDead) const {
491 bool Found = false;
492 for (const MachineOperand &MO : MI.operands()) {
493 bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(PhysReg: ARM::CPSR);
494 bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR;
495 if (ClobbersCPSR || IsCPSR) {
496
497 // Filter out T1 instructions that have a dead CPSR,
498 // allowing IT blocks to be generated containing T1 instructions
499 const MCInstrDesc &MCID = MI.getDesc();
500 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() &&
501 SkipDead)
502 continue;
503
504 Pred.push_back(x: MO);
505 Found = true;
506 }
507 }
508
509 return Found;
510}
511
512bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
513 for (const auto &MO : MI.operands())
514 if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
515 return true;
516 return false;
517}
518
519static bool isEligibleForITBlock(const MachineInstr *MI) {
520 switch (MI->getOpcode()) {
521 default: return true;
522 case ARM::tADC: // ADC (register) T1
523 case ARM::tADDi3: // ADD (immediate) T1
524 case ARM::tADDi8: // ADD (immediate) T2
525 case ARM::tADDrr: // ADD (register) T1
526 case ARM::tAND: // AND (register) T1
527 case ARM::tASRri: // ASR (immediate) T1
528 case ARM::tASRrr: // ASR (register) T1
529 case ARM::tBIC: // BIC (register) T1
530 case ARM::tEOR: // EOR (register) T1
531 case ARM::tLSLri: // LSL (immediate) T1
532 case ARM::tLSLrr: // LSL (register) T1
533 case ARM::tLSRri: // LSR (immediate) T1
534 case ARM::tLSRrr: // LSR (register) T1
535 case ARM::tMUL: // MUL T1
536 case ARM::tMVN: // MVN (register) T1
537 case ARM::tORR: // ORR (register) T1
538 case ARM::tROR: // ROR (register) T1
539 case ARM::tRSB: // RSB (immediate) T1
540 case ARM::tSBC: // SBC (register) T1
541 case ARM::tSUBi3: // SUB (immediate) T1
542 case ARM::tSUBi8: // SUB (immediate) T2
543 case ARM::tSUBrr: // SUB (register) T1
544 return !ARMBaseInstrInfo::isCPSRDefined(MI: *MI);
545 }
546}
547
548/// isPredicable - Return true if the specified instruction can be predicated.
549/// By default, this returns true for every instruction with a
550/// PredicateOperand.
551bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
552 if (!MI.isPredicable())
553 return false;
554
555 if (MI.isBundle())
556 return false;
557
558 if (!isEligibleForITBlock(MI: &MI))
559 return false;
560
561 const MachineFunction *MF = MI.getParent()->getParent();
562 const ARMFunctionInfo *AFI =
563 MF->getInfo<ARMFunctionInfo>();
564
565 // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
566 // In their ARM encoding, they can't be encoded in a conditional form.
567 if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
568 return false;
569
570 // Make indirect control flow changes unpredictable when SLS mitigation is
571 // enabled.
572 const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>();
573 if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI))
574 return false;
575 if (ST.hardenSlsBlr() && isIndirectCall(MI))
576 return false;
577
578 if (AFI->isThumb2Function()) {
579 if (getSubtarget().restrictIT())
580 return isV8EligibleForIT(Instr: &MI);
581 }
582
583 return true;
584}
585
586namespace llvm {
587
588template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
589 for (const MachineOperand &MO : MI->operands()) {
590 if (!MO.isReg() || MO.isUndef() || MO.isUse())
591 continue;
592 if (MO.getReg() != ARM::CPSR)
593 continue;
594 if (!MO.isDead())
595 return false;
596 }
597 // all definitions of CPSR are dead
598 return true;
599}
600
601} // end namespace llvm
602
603/// GetInstSize - Return the size of the specified MachineInstr.
604///
605unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
606 const MachineBasicBlock &MBB = *MI.getParent();
607 const MachineFunction *MF = MBB.getParent();
608 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
609
610 const MCInstrDesc &MCID = MI.getDesc();
611
612 switch (MI.getOpcode()) {
613 default:
614 // Return the size specified in .td file. If there's none, return 0, as we
615 // can't define a default size (Thumb1 instructions are 2 bytes, Thumb2
616 // instructions are 2-4 bytes, and ARM instructions are 4 bytes), in
617 // contrast to AArch64 instructions which have a default size of 4 bytes for
618 // example.
619 return MCID.getSize();
620 case TargetOpcode::BUNDLE:
621 return getInstBundleSize(MI);
622 case TargetOpcode::COPY:
623 if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
624 return 4;
625 else
626 return 2;
627 case ARM::CONSTPOOL_ENTRY:
628 case ARM::JUMPTABLE_INSTS:
629 case ARM::JUMPTABLE_ADDRS:
630 case ARM::JUMPTABLE_TBB:
631 case ARM::JUMPTABLE_TBH:
632 // If this machine instr is a constant pool entry, its size is recorded as
633 // operand #2.
634 return MI.getOperand(i: 2).getImm();
635 case ARM::SPACE:
636 return MI.getOperand(i: 1).getImm();
637 case ARM::INLINEASM:
638 case ARM::INLINEASM_BR: {
639 // If this machine instr is an inline asm, measure it.
640 unsigned Size = getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
641 if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
642 Size = alignTo(Value: Size, Align: 4);
643 return Size;
644 }
645 }
646}
647
648void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
649 MachineBasicBlock::iterator I,
650 MCRegister DestReg, bool KillSrc,
651 const ARMSubtarget &Subtarget) const {
652 unsigned Opc = Subtarget.isThumb()
653 ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
654 : ARM::MRS;
655
656 MachineInstrBuilder MIB =
657 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc), DestReg);
658
659 // There is only 1 A/R class MRS instruction, and it always refers to
660 // APSR. However, there are lots of other possibilities on M-class cores.
661 if (Subtarget.isMClass())
662 MIB.addImm(Val: 0x800);
663
664 MIB.add(MOs: predOps(Pred: ARMCC::AL))
665 .addReg(RegNo: ARM::CPSR, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
666}
667
668void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
669 MachineBasicBlock::iterator I,
670 MCRegister SrcReg, bool KillSrc,
671 const ARMSubtarget &Subtarget) const {
672 unsigned Opc = Subtarget.isThumb()
673 ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
674 : ARM::MSR;
675
676 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc));
677
678 if (Subtarget.isMClass())
679 MIB.addImm(Val: 0x800);
680 else
681 MIB.addImm(Val: 8);
682
683 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
684 .add(MOs: predOps(Pred: ARMCC::AL))
685 .addReg(RegNo: ARM::CPSR, Flags: RegState::Implicit | RegState::Define);
686}
687
688void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
689 MIB.addImm(Val: ARMVCC::None);
690 MIB.addReg(RegNo: 0);
691 MIB.addReg(RegNo: 0); // tp_reg
692}
693
694void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
695 Register DestReg) {
696 addUnpredicatedMveVpredNOp(MIB);
697 MIB.addReg(RegNo: DestReg, Flags: RegState::Undef);
698}
699
700void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
701 MIB.addImm(Val: Cond);
702 MIB.addReg(RegNo: ARM::VPR, Flags: RegState::Implicit);
703 MIB.addReg(RegNo: 0); // tp_reg
704}
705
706void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
707 unsigned Cond, unsigned Inactive) {
708 addPredicatedMveVpredNOp(MIB, Cond);
709 MIB.addReg(RegNo: Inactive);
710}
711
712void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
713 MachineBasicBlock::iterator I,
714 const DebugLoc &DL, Register DestReg,
715 Register SrcReg, bool KillSrc,
716 bool RenamableDest,
717 bool RenamableSrc) const {
718 bool GPRDest = ARM::GPRRegClass.contains(Reg: DestReg);
719 bool GPRSrc = ARM::GPRRegClass.contains(Reg: SrcReg);
720
721 if (GPRDest && GPRSrc) {
722 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MOVr), DestReg)
723 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
724 .add(MOs: predOps(Pred: ARMCC::AL))
725 .add(MO: condCodeOp());
726 return;
727 }
728
729 bool SPRDest = ARM::SPRRegClass.contains(Reg: DestReg);
730 bool SPRSrc = ARM::SPRRegClass.contains(Reg: SrcReg);
731
732 unsigned Opc = 0;
733 if (SPRDest && SPRSrc)
734 Opc = ARM::VMOVS;
735 else if (GPRDest && SPRSrc)
736 Opc = ARM::VMOVRS;
737 else if (SPRDest && GPRSrc)
738 Opc = ARM::VMOVSR;
739 else if (ARM::DPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg) && Subtarget.hasFP64())
740 Opc = ARM::VMOVD;
741 else if (ARM::QPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
742 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MQPRCopy;
743
744 if (Opc) {
745 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg);
746 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
747 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
748 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
749 if (Opc == ARM::MVE_VORR)
750 addUnpredicatedMveVpredROp(MIB, DestReg);
751 else if (Opc != ARM::MQPRCopy)
752 MIB.add(MOs: predOps(Pred: ARMCC::AL));
753 return;
754 }
755
756 // Handle register classes that require multiple instructions.
757 unsigned BeginIdx = 0;
758 unsigned SubRegs = 0;
759 int Spacing = 1;
760
761 // Use VORRq when possible.
762 if (ARM::QQPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
763 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
764 BeginIdx = ARM::qsub_0;
765 SubRegs = 2;
766 } else if (ARM::QQQQPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
767 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
768 BeginIdx = ARM::qsub_0;
769 SubRegs = 4;
770 // Fall back to VMOVD.
771 } else if (ARM::DPairRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
772 Opc = ARM::VMOVD;
773 BeginIdx = ARM::dsub_0;
774 SubRegs = 2;
775 } else if (ARM::DTripleRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
776 Opc = ARM::VMOVD;
777 BeginIdx = ARM::dsub_0;
778 SubRegs = 3;
779 } else if (ARM::DQuadRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
780 Opc = ARM::VMOVD;
781 BeginIdx = ARM::dsub_0;
782 SubRegs = 4;
783 } else if (ARM::GPRPairRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
784 Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
785 BeginIdx = ARM::gsub_0;
786 SubRegs = 2;
787 } else if (ARM::DPairSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
788 Opc = ARM::VMOVD;
789 BeginIdx = ARM::dsub_0;
790 SubRegs = 2;
791 Spacing = 2;
792 } else if (ARM::DTripleSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
793 Opc = ARM::VMOVD;
794 BeginIdx = ARM::dsub_0;
795 SubRegs = 3;
796 Spacing = 2;
797 } else if (ARM::DQuadSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
798 Opc = ARM::VMOVD;
799 BeginIdx = ARM::dsub_0;
800 SubRegs = 4;
801 Spacing = 2;
802 } else if (ARM::DPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg) &&
803 !Subtarget.hasFP64()) {
804 Opc = ARM::VMOVS;
805 BeginIdx = ARM::ssub_0;
806 SubRegs = 2;
807 } else if (SrcReg == ARM::CPSR) {
808 copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
809 return;
810 } else if (DestReg == ARM::CPSR) {
811 copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
812 return;
813 } else if (DestReg == ARM::VPR) {
814 assert(ARM::GPRRegClass.contains(SrcReg));
815 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMSR_P0), DestReg)
816 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
817 .add(MOs: predOps(Pred: ARMCC::AL));
818 return;
819 } else if (SrcReg == ARM::VPR) {
820 assert(ARM::GPRRegClass.contains(DestReg));
821 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMRS_P0), DestReg)
822 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
823 .add(MOs: predOps(Pred: ARMCC::AL));
824 return;
825 } else if (DestReg == ARM::FPSCR_NZCV) {
826 assert(ARM::GPRRegClass.contains(SrcReg));
827 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMSR_FPSCR_NZCVQC), DestReg)
828 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
829 .add(MOs: predOps(Pred: ARMCC::AL));
830 return;
831 } else if (SrcReg == ARM::FPSCR_NZCV) {
832 assert(ARM::GPRRegClass.contains(DestReg));
833 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMRS_FPSCR_NZCVQC), DestReg)
834 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
835 .add(MOs: predOps(Pred: ARMCC::AL));
836 return;
837 }
838
839 assert(Opc && "Impossible reg-to-reg copy");
840
841 const TargetRegisterInfo *TRI = &getRegisterInfo();
842 MachineInstrBuilder Mov;
843
844 // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
845 if (TRI->regsOverlap(RegA: SrcReg, RegB: TRI->getSubReg(Reg: DestReg, Idx: BeginIdx))) {
846 BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
847 Spacing = -Spacing;
848 }
849#ifndef NDEBUG
850 SmallSet<unsigned, 4> DstRegs;
851#endif
852 for (unsigned i = 0; i != SubRegs; ++i) {
853 Register Dst = TRI->getSubReg(Reg: DestReg, Idx: BeginIdx + i * Spacing);
854 Register Src = TRI->getSubReg(Reg: SrcReg, Idx: BeginIdx + i * Spacing);
855 assert(Dst && Src && "Bad sub-register");
856#ifndef NDEBUG
857 assert(!DstRegs.count(Src) && "destructive vector copy");
858 DstRegs.insert(Dst);
859#endif
860 Mov = BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc), DestReg: Dst).addReg(RegNo: Src);
861 // VORR (NEON or MVE) takes two source operands.
862 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
863 Mov.addReg(RegNo: Src);
864 }
865 // MVE VORR takes predicate operands in place of an ordinary condition.
866 if (Opc == ARM::MVE_VORR)
867 addUnpredicatedMveVpredROp(MIB&: Mov, DestReg: Dst);
868 else
869 Mov = Mov.add(MOs: predOps(Pred: ARMCC::AL));
870 // MOVr can set CC.
871 if (Opc == ARM::MOVr)
872 Mov = Mov.add(MO: condCodeOp());
873 }
874 // Add implicit super-register defs and kills to the last instruction.
875 Mov->addRegisterDefined(Reg: DestReg, RegInfo: TRI);
876 if (KillSrc)
877 Mov->addRegisterKilled(IncomingReg: SrcReg, RegInfo: TRI);
878}
879
880std::optional<DestSourcePair>
881ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
882 // VMOVRRD is also a copy instruction but it requires
883 // special way of handling. It is more complex copy version
884 // and since that we are not considering it. For recognition
885 // of such instruction isExtractSubregLike MI interface function
886 // could be used.
887 // VORRq is considered as a move only if two inputs are
888 // the same register.
889 if (!MI.isMoveReg() ||
890 (MI.getOpcode() == ARM::VORRq &&
891 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg()))
892 return std::nullopt;
893 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
894}
895
896std::optional<ParamLoadedValue>
897ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
898 Register Reg) const {
899 if (auto DstSrcPair = isCopyInstrImpl(MI)) {
900 Register DstReg = DstSrcPair->Destination->getReg();
901
902 // TODO: We don't handle cases where the forwarding reg is narrower/wider
903 // than the copy registers. Consider for example:
904 //
905 // s16 = VMOVS s0
906 // s17 = VMOVS s1
907 // call @callee(d0)
908 //
909 // We'd like to describe the call site value of d0 as d8, but this requires
910 // gathering and merging the descriptions for the two VMOVS instructions.
911 //
912 // We also don't handle the reverse situation, where the forwarding reg is
913 // narrower than the copy destination:
914 //
915 // d8 = VMOVD d0
916 // call @callee(s1)
917 //
918 // We need to produce a fragment description (the call site value of s1 is
919 // /not/ just d8).
920 if (DstReg != Reg)
921 return std::nullopt;
922 }
923 return TargetInstrInfo::describeLoadedValue(MI, Reg);
924}
925
926const MachineInstrBuilder &ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB,
927 unsigned Reg,
928 unsigned SubIdx,
929 RegState State) const {
930 if (!SubIdx)
931 return MIB.addReg(RegNo: Reg, Flags: State);
932
933 if (Register::isPhysicalRegister(Reg))
934 return MIB.addReg(RegNo: getRegisterInfo().getSubReg(Reg, Idx: SubIdx), Flags: State);
935 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
936}
937
938void ARMBaseInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
939 MachineBasicBlock::iterator I,
940 Register SrcReg, bool isKill, int FI,
941 const TargetRegisterClass *RC,
942 Register VReg,
943 MachineInstr::MIFlag Flags) const {
944 MachineFunction &MF = *MBB.getParent();
945 MachineFrameInfo &MFI = MF.getFrameInfo();
946 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
947 const ARMBaseRegisterInfo &TRI = getRegisterInfo();
948
949 MachineMemOperand *MMO = MF.getMachineMemOperand(
950 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI), F: MachineMemOperand::MOStore,
951 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
952
953 switch (TRI.getSpillSize(RC: *RC)) {
954 case 2:
955 if (ARM::HPRRegClass.hasSubClassEq(RC)) {
956 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRH))
957 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
958 .addFrameIndex(Idx: FI)
959 .addImm(Val: 0)
960 .addMemOperand(MMO)
961 .add(MOs: predOps(Pred: ARMCC::AL));
962 } else
963 llvm_unreachable("Unknown reg class!");
964 break;
965 case 4:
966 if (ARM::GPRRegClass.hasSubClassEq(RC)) {
967 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STRi12))
968 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
969 .addFrameIndex(Idx: FI)
970 .addImm(Val: 0)
971 .addMemOperand(MMO)
972 .add(MOs: predOps(Pred: ARMCC::AL));
973 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
974 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRS))
975 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
976 .addFrameIndex(Idx: FI)
977 .addImm(Val: 0)
978 .addMemOperand(MMO)
979 .add(MOs: predOps(Pred: ARMCC::AL));
980 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
981 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTR_P0_off))
982 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
983 .addFrameIndex(Idx: FI)
984 .addImm(Val: 0)
985 .addMemOperand(MMO)
986 .add(MOs: predOps(Pred: ARMCC::AL));
987 } else if (ARM::cl_FPSCR_NZCVRegClass.hasSubClassEq(RC)) {
988 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTR_FPSCR_NZCVQC_off))
989 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
990 .addFrameIndex(Idx: FI)
991 .addImm(Val: 0)
992 .addMemOperand(MMO)
993 .add(MOs: predOps(Pred: ARMCC::AL));
994 } else
995 llvm_unreachable("Unknown reg class!");
996 break;
997 case 8:
998 if (ARM::DPRRegClass.hasSubClassEq(RC)) {
999 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRD))
1000 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1001 .addFrameIndex(Idx: FI)
1002 .addImm(Val: 0)
1003 .addMemOperand(MMO)
1004 .add(MOs: predOps(Pred: ARMCC::AL));
1005 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1006 if (Subtarget.hasV5TEOps()) {
1007 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STRD));
1008 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_0, State: getKillRegState(B: isKill));
1009 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_1, State: {});
1010 MIB.addFrameIndex(Idx: FI).addReg(RegNo: 0).addImm(Val: 0).addMemOperand(MMO)
1011 .add(MOs: predOps(Pred: ARMCC::AL));
1012 } else {
1013 // Fallback to STM instruction, which has existed since the dawn of
1014 // time.
1015 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STMIA))
1016 .addFrameIndex(Idx: FI)
1017 .addMemOperand(MMO)
1018 .add(MOs: predOps(Pred: ARMCC::AL));
1019 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_0, State: getKillRegState(B: isKill));
1020 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_1, State: {});
1021 }
1022 } else
1023 llvm_unreachable("Unknown reg class!");
1024 break;
1025 case 16:
1026 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1027 // Use aligned spills if the stack can be realigned.
1028 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1029 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1q64))
1030 .addFrameIndex(Idx: FI)
1031 .addImm(Val: 16)
1032 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1033 .addMemOperand(MMO)
1034 .add(MOs: predOps(Pred: ARMCC::AL));
1035 } else {
1036 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTMQIA))
1037 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1038 .addFrameIndex(Idx: FI)
1039 .addMemOperand(MMO)
1040 .add(MOs: predOps(Pred: ARMCC::AL));
1041 }
1042 } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1043 Subtarget.hasMVEIntegerOps()) {
1044 auto MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MVE_VSTRWU32));
1045 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1046 .addFrameIndex(Idx: FI)
1047 .addImm(Val: 0)
1048 .addMemOperand(MMO);
1049 addUnpredicatedMveVpredNOp(MIB);
1050 } else
1051 llvm_unreachable("Unknown reg class!");
1052 break;
1053 case 24:
1054 if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1055 // Use aligned spills if the stack can be realigned.
1056 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1057 Subtarget.hasNEON()) {
1058 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1d64TPseudo))
1059 .addFrameIndex(Idx: FI)
1060 .addImm(Val: 16)
1061 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1062 .addMemOperand(MMO)
1063 .add(MOs: predOps(Pred: ARMCC::AL));
1064 } else {
1065 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(),
1066 MCID: get(Opcode: ARM::VSTMDIA))
1067 .addFrameIndex(Idx: FI)
1068 .add(MOs: predOps(Pred: ARMCC::AL))
1069 .addMemOperand(MMO);
1070 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1071 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1072 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1073 }
1074 } else
1075 llvm_unreachable("Unknown reg class!");
1076 break;
1077 case 32:
1078 if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1079 ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1080 ARM::DQuadRegClass.hasSubClassEq(RC)) {
1081 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1082 Subtarget.hasNEON()) {
1083 // FIXME: It's possible to only store part of the QQ register if the
1084 // spilled def has a sub-register index.
1085 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1d64QPseudo))
1086 .addFrameIndex(Idx: FI)
1087 .addImm(Val: 16)
1088 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1089 .addMemOperand(MMO)
1090 .add(MOs: predOps(Pred: ARMCC::AL));
1091 } else if (Subtarget.hasMVEIntegerOps()) {
1092 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MQQPRStore))
1093 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1094 .addFrameIndex(Idx: FI)
1095 .addMemOperand(MMO);
1096 } else {
1097 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(),
1098 MCID: get(Opcode: ARM::VSTMDIA))
1099 .addFrameIndex(Idx: FI)
1100 .add(MOs: predOps(Pred: ARMCC::AL))
1101 .addMemOperand(MMO);
1102 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1103 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1104 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1105 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_3, State: {});
1106 }
1107 } else
1108 llvm_unreachable("Unknown reg class!");
1109 break;
1110 case 64:
1111 if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1112 Subtarget.hasMVEIntegerOps()) {
1113 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MQQQQPRStore))
1114 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1115 .addFrameIndex(Idx: FI)
1116 .addMemOperand(MMO);
1117 } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1118 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTMDIA))
1119 .addFrameIndex(Idx: FI)
1120 .add(MOs: predOps(Pred: ARMCC::AL))
1121 .addMemOperand(MMO);
1122 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1123 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1124 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1125 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_3, State: {});
1126 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_4, State: {});
1127 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_5, State: {});
1128 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_6, State: {});
1129 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_7, State: {});
1130 } else
1131 llvm_unreachable("Unknown reg class!");
1132 break;
1133 default:
1134 llvm_unreachable("Unknown reg class!");
1135 }
1136}
1137
1138Register ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1139 int &FrameIndex) const {
1140 switch (MI.getOpcode()) {
1141 default: break;
1142 case ARM::STRrs:
1143 case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1144 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isReg() &&
1145 MI.getOperand(i: 3).isImm() && MI.getOperand(i: 2).getReg() == 0 &&
1146 MI.getOperand(i: 3).getImm() == 0) {
1147 FrameIndex = MI.getOperand(i: 1).getIndex();
1148 return MI.getOperand(i: 0).getReg();
1149 }
1150 break;
1151 case ARM::STRi12:
1152 case ARM::t2STRi12:
1153 case ARM::tSTRspi:
1154 case ARM::VSTRD:
1155 case ARM::VSTRS:
1156 case ARM::VSTRH:
1157 case ARM::VSTR_P0_off:
1158 case ARM::VSTR_FPSCR_NZCVQC_off:
1159 case ARM::MVE_VSTRWU32:
1160 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
1161 MI.getOperand(i: 2).getImm() == 0) {
1162 FrameIndex = MI.getOperand(i: 1).getIndex();
1163 return MI.getOperand(i: 0).getReg();
1164 }
1165 break;
1166 case ARM::VST1q64:
1167 case ARM::VST1d64TPseudo:
1168 case ARM::VST1d64QPseudo:
1169 if (MI.getOperand(i: 0).isFI() && MI.getOperand(i: 2).getSubReg() == 0) {
1170 FrameIndex = MI.getOperand(i: 0).getIndex();
1171 return MI.getOperand(i: 2).getReg();
1172 }
1173 break;
1174 case ARM::VSTMQIA:
1175 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1176 FrameIndex = MI.getOperand(i: 1).getIndex();
1177 return MI.getOperand(i: 0).getReg();
1178 }
1179 break;
1180 case ARM::MQQPRStore:
1181 case ARM::MQQQQPRStore:
1182 if (MI.getOperand(i: 1).isFI()) {
1183 FrameIndex = MI.getOperand(i: 1).getIndex();
1184 return MI.getOperand(i: 0).getReg();
1185 }
1186 break;
1187 }
1188
1189 return 0;
1190}
1191
1192Register ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1193 int &FrameIndex) const {
1194 SmallVector<const MachineMemOperand *, 1> Accesses;
1195 if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1196 Accesses.size() == 1) {
1197 FrameIndex =
1198 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
1199 ->getFrameIndex();
1200 return true;
1201 }
1202 return false;
1203}
1204
1205void ARMBaseInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1206 MachineBasicBlock::iterator I,
1207 Register DestReg, int FI,
1208 const TargetRegisterClass *RC,
1209 Register VReg, unsigned SubReg,
1210 MachineInstr::MIFlag Flags) const {
1211 DebugLoc DL;
1212 if (I != MBB.end()) DL = I->getDebugLoc();
1213 MachineFunction &MF = *MBB.getParent();
1214 MachineFrameInfo &MFI = MF.getFrameInfo();
1215 const Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
1216 MachineMemOperand *MMO = MF.getMachineMemOperand(
1217 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI), F: MachineMemOperand::MOLoad,
1218 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
1219
1220 const ARMBaseRegisterInfo &TRI = getRegisterInfo();
1221 switch (TRI.getSpillSize(RC: *RC)) {
1222 case 2:
1223 if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1224 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRH), DestReg)
1225 .addFrameIndex(Idx: FI)
1226 .addImm(Val: 0)
1227 .addMemOperand(MMO)
1228 .add(MOs: predOps(Pred: ARMCC::AL));
1229 } else
1230 llvm_unreachable("Unknown reg class!");
1231 break;
1232 case 4:
1233 if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1234 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDRi12), DestReg)
1235 .addFrameIndex(Idx: FI)
1236 .addImm(Val: 0)
1237 .addMemOperand(MMO)
1238 .add(MOs: predOps(Pred: ARMCC::AL));
1239 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1240 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRS), DestReg)
1241 .addFrameIndex(Idx: FI)
1242 .addImm(Val: 0)
1243 .addMemOperand(MMO)
1244 .add(MOs: predOps(Pred: ARMCC::AL));
1245 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1246 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDR_P0_off), DestReg)
1247 .addFrameIndex(Idx: FI)
1248 .addImm(Val: 0)
1249 .addMemOperand(MMO)
1250 .add(MOs: predOps(Pred: ARMCC::AL));
1251 } else if (ARM::cl_FPSCR_NZCVRegClass.hasSubClassEq(RC)) {
1252 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDR_FPSCR_NZCVQC_off), DestReg)
1253 .addFrameIndex(Idx: FI)
1254 .addImm(Val: 0)
1255 .addMemOperand(MMO)
1256 .add(MOs: predOps(Pred: ARMCC::AL));
1257 } else
1258 llvm_unreachable("Unknown reg class!");
1259 break;
1260 case 8:
1261 if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRD), DestReg)
1263 .addFrameIndex(Idx: FI)
1264 .addImm(Val: 0)
1265 .addMemOperand(MMO)
1266 .add(MOs: predOps(Pred: ARMCC::AL));
1267 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1268 MachineInstrBuilder MIB;
1269
1270 if (Subtarget.hasV5TEOps()) {
1271 MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDRD));
1272 AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_0, State: RegState::DefineNoRead);
1273 AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_1, State: RegState::DefineNoRead);
1274 MIB.addFrameIndex(Idx: FI).addReg(RegNo: 0).addImm(Val: 0).addMemOperand(MMO)
1275 .add(MOs: predOps(Pred: ARMCC::AL));
1276 } else {
1277 // Fallback to LDM instruction, which has existed since the dawn of
1278 // time.
1279 MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDMIA))
1280 .addFrameIndex(Idx: FI)
1281 .addMemOperand(MMO)
1282 .add(MOs: predOps(Pred: ARMCC::AL));
1283 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_0, State: RegState::DefineNoRead);
1284 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_1, State: RegState::DefineNoRead);
1285 }
1286
1287 if (DestReg.isPhysical())
1288 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1289 } else
1290 llvm_unreachable("Unknown reg class!");
1291 break;
1292 case 16:
1293 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1294 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1295 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1q64), DestReg)
1296 .addFrameIndex(Idx: FI)
1297 .addImm(Val: 16)
1298 .addMemOperand(MMO)
1299 .add(MOs: predOps(Pred: ARMCC::AL));
1300 } else {
1301 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMQIA), DestReg)
1302 .addFrameIndex(Idx: FI)
1303 .addMemOperand(MMO)
1304 .add(MOs: predOps(Pred: ARMCC::AL));
1305 }
1306 } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1307 Subtarget.hasMVEIntegerOps()) {
1308 auto MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MVE_VLDRWU32), DestReg);
1309 MIB.addFrameIndex(Idx: FI)
1310 .addImm(Val: 0)
1311 .addMemOperand(MMO);
1312 addUnpredicatedMveVpredNOp(MIB);
1313 } else
1314 llvm_unreachable("Unknown reg class!");
1315 break;
1316 case 24:
1317 if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1318 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1319 Subtarget.hasNEON()) {
1320 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1d64TPseudo), DestReg)
1321 .addFrameIndex(Idx: FI)
1322 .addImm(Val: 16)
1323 .addMemOperand(MMO)
1324 .add(MOs: predOps(Pred: ARMCC::AL));
1325 } else {
1326 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1327 .addFrameIndex(Idx: FI)
1328 .addMemOperand(MMO)
1329 .add(MOs: predOps(Pred: ARMCC::AL));
1330 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1331 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1332 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1333 if (DestReg.isPhysical())
1334 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1335 }
1336 } else
1337 llvm_unreachable("Unknown reg class!");
1338 break;
1339 case 32:
1340 if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1341 ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1342 ARM::DQuadRegClass.hasSubClassEq(RC)) {
1343 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1344 Subtarget.hasNEON()) {
1345 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1d64QPseudo), DestReg)
1346 .addFrameIndex(Idx: FI)
1347 .addImm(Val: 16)
1348 .addMemOperand(MMO)
1349 .add(MOs: predOps(Pred: ARMCC::AL));
1350 } else if (Subtarget.hasMVEIntegerOps()) {
1351 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MQQPRLoad), DestReg)
1352 .addFrameIndex(Idx: FI)
1353 .addMemOperand(MMO);
1354 } else {
1355 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1356 .addFrameIndex(Idx: FI)
1357 .add(MOs: predOps(Pred: ARMCC::AL))
1358 .addMemOperand(MMO);
1359 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1360 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1361 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1362 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_3, State: RegState::DefineNoRead);
1363 if (DestReg.isPhysical())
1364 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1365 }
1366 } else
1367 llvm_unreachable("Unknown reg class!");
1368 break;
1369 case 64:
1370 if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1371 Subtarget.hasMVEIntegerOps()) {
1372 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MQQQQPRLoad), DestReg)
1373 .addFrameIndex(Idx: FI)
1374 .addMemOperand(MMO);
1375 } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1376 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1377 .addFrameIndex(Idx: FI)
1378 .add(MOs: predOps(Pred: ARMCC::AL))
1379 .addMemOperand(MMO);
1380 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1381 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1382 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1383 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_3, State: RegState::DefineNoRead);
1384 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_4, State: RegState::DefineNoRead);
1385 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_5, State: RegState::DefineNoRead);
1386 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_6, State: RegState::DefineNoRead);
1387 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_7, State: RegState::DefineNoRead);
1388 if (DestReg.isPhysical())
1389 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1390 } else
1391 llvm_unreachable("Unknown reg class!");
1392 break;
1393 default:
1394 llvm_unreachable("Unknown regclass!");
1395 }
1396}
1397
1398Register ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1399 int &FrameIndex) const {
1400 switch (MI.getOpcode()) {
1401 default: break;
1402 case ARM::LDRrs:
1403 case ARM::t2LDRs: // FIXME: don't use t2LDRs to access frame.
1404 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isReg() &&
1405 MI.getOperand(i: 3).isImm() && MI.getOperand(i: 2).getReg() == 0 &&
1406 MI.getOperand(i: 3).getImm() == 0) {
1407 FrameIndex = MI.getOperand(i: 1).getIndex();
1408 return MI.getOperand(i: 0).getReg();
1409 }
1410 break;
1411 case ARM::LDRi12:
1412 case ARM::t2LDRi12:
1413 case ARM::tLDRspi:
1414 case ARM::VLDRD:
1415 case ARM::VLDRS:
1416 case ARM::VLDRH:
1417 case ARM::VLDR_P0_off:
1418 case ARM::VLDR_FPSCR_NZCVQC_off:
1419 case ARM::MVE_VLDRWU32:
1420 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
1421 MI.getOperand(i: 2).getImm() == 0) {
1422 FrameIndex = MI.getOperand(i: 1).getIndex();
1423 return MI.getOperand(i: 0).getReg();
1424 }
1425 break;
1426 case ARM::VLD1q64:
1427 case ARM::VLD1d8TPseudo:
1428 case ARM::VLD1d16TPseudo:
1429 case ARM::VLD1d32TPseudo:
1430 case ARM::VLD1d64TPseudo:
1431 case ARM::VLD1d8QPseudo:
1432 case ARM::VLD1d16QPseudo:
1433 case ARM::VLD1d32QPseudo:
1434 case ARM::VLD1d64QPseudo:
1435 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1436 FrameIndex = MI.getOperand(i: 1).getIndex();
1437 return MI.getOperand(i: 0).getReg();
1438 }
1439 break;
1440 case ARM::VLDMQIA:
1441 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1442 FrameIndex = MI.getOperand(i: 1).getIndex();
1443 return MI.getOperand(i: 0).getReg();
1444 }
1445 break;
1446 case ARM::MQQPRLoad:
1447 case ARM::MQQQQPRLoad:
1448 if (MI.getOperand(i: 1).isFI()) {
1449 FrameIndex = MI.getOperand(i: 1).getIndex();
1450 return MI.getOperand(i: 0).getReg();
1451 }
1452 break;
1453 }
1454
1455 return 0;
1456}
1457
1458Register ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1459 int &FrameIndex) const {
1460 SmallVector<const MachineMemOperand *, 1> Accesses;
1461 if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1462 Accesses.size() == 1) {
1463 FrameIndex =
1464 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
1465 ->getFrameIndex();
1466 return true;
1467 }
1468 return false;
1469}
1470
1471/// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1472/// depending on whether the result is used.
1473void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1474 bool isThumb1 = Subtarget.isThumb1Only();
1475 bool isThumb2 = Subtarget.isThumb2();
1476 const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1477
1478 DebugLoc dl = MI->getDebugLoc();
1479 MachineBasicBlock *BB = MI->getParent();
1480
1481 MachineInstrBuilder LDM, STM;
1482 if (isThumb1 || !MI->getOperand(i: 1).isDead()) {
1483 MachineOperand LDWb(MI->getOperand(i: 1));
1484 LDM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2LDMIA_UPD
1485 : isThumb1 ? ARM::tLDMIA_UPD
1486 : ARM::LDMIA_UPD))
1487 .add(MO: LDWb);
1488 } else {
1489 LDM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1490 }
1491
1492 if (isThumb1 || !MI->getOperand(i: 0).isDead()) {
1493 MachineOperand STWb(MI->getOperand(i: 0));
1494 STM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2STMIA_UPD
1495 : isThumb1 ? ARM::tSTMIA_UPD
1496 : ARM::STMIA_UPD))
1497 .add(MO: STWb);
1498 } else {
1499 STM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1500 }
1501
1502 MachineOperand LDBase(MI->getOperand(i: 3));
1503 LDM.add(MO: LDBase).add(MOs: predOps(Pred: ARMCC::AL));
1504
1505 MachineOperand STBase(MI->getOperand(i: 2));
1506 STM.add(MO: STBase).add(MOs: predOps(Pred: ARMCC::AL));
1507
1508 // Sort the scratch registers into ascending order.
1509 const TargetRegisterInfo &TRI = getRegisterInfo();
1510 SmallVector<unsigned, 6> ScratchRegs;
1511 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 5))
1512 ScratchRegs.push_back(Elt: MO.getReg());
1513 llvm::sort(C&: ScratchRegs,
1514 Comp: [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1515 return TRI.getEncodingValue(Reg: Reg1) <
1516 TRI.getEncodingValue(Reg: Reg2);
1517 });
1518
1519 for (const auto &Reg : ScratchRegs) {
1520 LDM.addReg(RegNo: Reg, Flags: RegState::Define);
1521 STM.addReg(RegNo: Reg, Flags: RegState::Kill);
1522 }
1523
1524 BB->erase(I: MI);
1525}
1526
1527bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1528 if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1529 expandLoadStackGuard(MI);
1530 MI.getParent()->erase(I: MI);
1531 return true;
1532 }
1533
1534 if (MI.getOpcode() == ARM::MEMCPY) {
1535 expandMEMCPY(MI);
1536 return true;
1537 }
1538
1539 // This hook gets to expand COPY instructions before they become
1540 // copyPhysReg() calls. Look for VMOVS instructions that can legally be
1541 // widened to VMOVD. We prefer the VMOVD when possible because it may be
1542 // changed into a VORR that can go down the NEON pipeline.
1543 if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1544 return false;
1545
1546 // Look for a copy between even S-registers. That is where we keep floats
1547 // when using NEON v2f32 instructions for f32 arithmetic.
1548 Register DstRegS = MI.getOperand(i: 0).getReg();
1549 Register SrcRegS = MI.getOperand(i: 1).getReg();
1550 if (!ARM::SPRRegClass.contains(Reg1: DstRegS, Reg2: SrcRegS))
1551 return false;
1552
1553 const TargetRegisterInfo *TRI = &getRegisterInfo();
1554 MCRegister DstRegD =
1555 TRI->getMatchingSuperReg(Reg: DstRegS, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
1556 MCRegister SrcRegD =
1557 TRI->getMatchingSuperReg(Reg: SrcRegS, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
1558 if (!DstRegD || !SrcRegD)
1559 return false;
1560
1561 // We want to widen this into a DstRegD = VMOVD SrcRegD copy. This is only
1562 // legal if the COPY already defines the full DstRegD, and it isn't a
1563 // sub-register insertion.
1564 if (!MI.definesRegister(Reg: DstRegD, TRI) || MI.readsRegister(Reg: DstRegD, TRI))
1565 return false;
1566
1567 // A dead copy shouldn't show up here, but reject it just in case.
1568 if (MI.getOperand(i: 0).isDead())
1569 return false;
1570
1571 // All clear, widen the COPY.
1572 LLVM_DEBUG(dbgs() << "widening: " << MI);
1573 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1574
1575 // Get rid of the old implicit-def of DstRegD. Leave it if it defines a Q-reg
1576 // or some other super-register.
1577 int ImpDefIdx = MI.findRegisterDefOperandIdx(Reg: DstRegD, /*TRI=*/nullptr);
1578 if (ImpDefIdx != -1)
1579 MI.removeOperand(OpNo: ImpDefIdx);
1580
1581 // Change the opcode and operands.
1582 MI.setDesc(get(Opcode: ARM::VMOVD));
1583 MI.getOperand(i: 0).setReg(DstRegD);
1584 MI.getOperand(i: 1).setReg(SrcRegD);
1585 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1586
1587 // We are now reading SrcRegD instead of SrcRegS. This may upset the
1588 // register scavenger and machine verifier, so we need to indicate that we
1589 // are reading an undefined value from SrcRegD, but a proper value from
1590 // SrcRegS.
1591 MI.getOperand(i: 1).setIsUndef();
1592 MIB.addReg(RegNo: SrcRegS, Flags: RegState::Implicit);
1593
1594 // SrcRegD may actually contain an unrelated value in the ssub_1
1595 // sub-register. Don't kill it. Only kill the ssub_0 sub-register.
1596 if (MI.getOperand(i: 1).isKill()) {
1597 MI.getOperand(i: 1).setIsKill(false);
1598 MI.addRegisterKilled(IncomingReg: SrcRegS, RegInfo: TRI, AddIfNotFound: true);
1599 }
1600
1601 LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1602 return true;
1603}
1604
1605/// Create a copy of a const pool value. Update CPI to the new index and return
1606/// the label UID.
1607static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1608 MachineConstantPool *MCP = MF.getConstantPool();
1609 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1610
1611 const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1612 assert(MCPE.isMachineConstantPoolEntry() &&
1613 "Expecting a machine constantpool entry!");
1614 ARMConstantPoolValue *ACPV =
1615 static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1616
1617 unsigned PCLabelId = AFI->createPICLabelUId();
1618 ARMConstantPoolValue *NewCPV = nullptr;
1619
1620 // FIXME: The below assumes PIC relocation model and that the function
1621 // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1622 // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1623 // instructions, so that's probably OK, but is PIC always correct when
1624 // we get here?
1625 if (ACPV->isGlobalValue())
1626 NewCPV = ARMConstantPoolConstant::Create(
1627 C: cast<ARMConstantPoolConstant>(Val: ACPV)->getGV(), ID: PCLabelId, Kind: ARMCP::CPValue,
1628 PCAdj: 4, Modifier: ACPV->getModifier(), AddCurrentAddress: ACPV->mustAddCurrentAddress());
1629 else if (ACPV->isExtSymbol())
1630 NewCPV = ARMConstantPoolSymbol::
1631 Create(C&: MF.getFunction().getContext(),
1632 s: cast<ARMConstantPoolSymbol>(Val: ACPV)->getSymbol(), ID: PCLabelId, PCAdj: 4);
1633 else if (ACPV->isBlockAddress())
1634 NewCPV = ARMConstantPoolConstant::
1635 Create(C: cast<ARMConstantPoolConstant>(Val: ACPV)->getBlockAddress(), ID: PCLabelId,
1636 Kind: ARMCP::CPBlockAddress, PCAdj: 4);
1637 else if (ACPV->isLSDA())
1638 NewCPV = ARMConstantPoolConstant::Create(C: &MF.getFunction(), ID: PCLabelId,
1639 Kind: ARMCP::CPLSDA, PCAdj: 4);
1640 else if (ACPV->isMachineBasicBlock())
1641 NewCPV = ARMConstantPoolMBB::
1642 Create(C&: MF.getFunction().getContext(),
1643 mbb: cast<ARMConstantPoolMBB>(Val: ACPV)->getMBB(), ID: PCLabelId, PCAdj: 4);
1644 else
1645 llvm_unreachable("Unexpected ARM constantpool value type!!");
1646 CPI = MCP->getConstantPoolIndex(V: NewCPV, Alignment: MCPE.getAlign());
1647 return PCLabelId;
1648}
1649
1650void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1651 MachineBasicBlock::iterator I,
1652 Register DestReg, unsigned SubIdx,
1653 const MachineInstr &Orig,
1654 LaneBitmask UsedLanes) const {
1655 unsigned Opcode = Orig.getOpcode();
1656 switch (Opcode) {
1657 default: {
1658 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
1659 MI->substituteRegister(FromReg: Orig.getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
1660 MBB.insert(I, MI);
1661 break;
1662 }
1663 case ARM::tLDRpci_pic:
1664 case ARM::t2LDRpci_pic: {
1665 MachineFunction &MF = *MBB.getParent();
1666 unsigned CPI = Orig.getOperand(i: 1).getIndex();
1667 unsigned PCLabelId = duplicateCPV(MF, CPI);
1668 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode), DestReg)
1669 .addConstantPoolIndex(Idx: CPI)
1670 .addImm(Val: PCLabelId)
1671 .cloneMemRefs(OtherMI: Orig);
1672 break;
1673 }
1674 }
1675}
1676
1677MachineInstr &
1678ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1679 MachineBasicBlock::iterator InsertBefore,
1680 const MachineInstr &Orig) const {
1681 MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1682 MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1683 for (;;) {
1684 switch (I->getOpcode()) {
1685 case ARM::tLDRpci_pic:
1686 case ARM::t2LDRpci_pic: {
1687 MachineFunction &MF = *MBB.getParent();
1688 unsigned CPI = I->getOperand(i: 1).getIndex();
1689 unsigned PCLabelId = duplicateCPV(MF, CPI);
1690 I->getOperand(i: 1).setIndex(CPI);
1691 I->getOperand(i: 2).setImm(PCLabelId);
1692 break;
1693 }
1694 }
1695 if (!I->isBundledWithSucc())
1696 break;
1697 ++I;
1698 }
1699 return Cloned;
1700}
1701
1702bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1703 const MachineInstr &MI1,
1704 const MachineRegisterInfo *MRI) const {
1705 unsigned Opcode = MI0.getOpcode();
1706 if (Opcode == ARM::t2LDRpci || Opcode == ARM::t2LDRpci_pic ||
1707 Opcode == ARM::tLDRpci || Opcode == ARM::tLDRpci_pic ||
1708 Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1709 Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1710 Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1711 Opcode == ARM::t2MOV_ga_pcrel) {
1712 if (MI1.getOpcode() != Opcode)
1713 return false;
1714 if (MI0.getNumOperands() != MI1.getNumOperands())
1715 return false;
1716
1717 const MachineOperand &MO0 = MI0.getOperand(i: 1);
1718 const MachineOperand &MO1 = MI1.getOperand(i: 1);
1719 if (MO0.getOffset() != MO1.getOffset())
1720 return false;
1721
1722 if (Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1723 Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1724 Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1725 Opcode == ARM::t2MOV_ga_pcrel)
1726 // Ignore the PC labels.
1727 return MO0.getGlobal() == MO1.getGlobal();
1728
1729 const MachineFunction *MF = MI0.getParent()->getParent();
1730 const MachineConstantPool *MCP = MF->getConstantPool();
1731 int CPI0 = MO0.getIndex();
1732 int CPI1 = MO1.getIndex();
1733 const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1734 const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1735 bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1736 bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1737 if (isARMCP0 && isARMCP1) {
1738 ARMConstantPoolValue *ACPV0 =
1739 static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1740 ARMConstantPoolValue *ACPV1 =
1741 static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1742 return ACPV0->hasSameValue(ACPV: ACPV1);
1743 } else if (!isARMCP0 && !isARMCP1) {
1744 return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1745 }
1746 return false;
1747 } else if (Opcode == ARM::PICLDR) {
1748 if (MI1.getOpcode() != Opcode)
1749 return false;
1750 if (MI0.getNumOperands() != MI1.getNumOperands())
1751 return false;
1752
1753 Register Addr0 = MI0.getOperand(i: 1).getReg();
1754 Register Addr1 = MI1.getOperand(i: 1).getReg();
1755 if (Addr0 != Addr1) {
1756 if (!MRI || !Addr0.isVirtual() || !Addr1.isVirtual())
1757 return false;
1758
1759 // This assumes SSA form.
1760 MachineInstr *Def0 = MRI->getVRegDef(Reg: Addr0);
1761 MachineInstr *Def1 = MRI->getVRegDef(Reg: Addr1);
1762 // Check if the loaded value, e.g. a constantpool of a global address, are
1763 // the same.
1764 if (!produceSameValue(MI0: *Def0, MI1: *Def1, MRI))
1765 return false;
1766 }
1767
1768 for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1769 // %12 = PICLDR %11, 0, 14, %noreg
1770 const MachineOperand &MO0 = MI0.getOperand(i);
1771 const MachineOperand &MO1 = MI1.getOperand(i);
1772 if (!MO0.isIdenticalTo(Other: MO1))
1773 return false;
1774 }
1775 return true;
1776 }
1777
1778 return MI0.isIdenticalTo(Other: MI1, Check: MachineInstr::IgnoreVRegDefs);
1779}
1780
1781/// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1782/// determine if two loads are loading from the same base address. It should
1783/// only return true if the base pointers are the same and the only differences
1784/// between the two addresses is the offset. It also returns the offsets by
1785/// reference.
1786///
1787/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1788/// is permanently disabled.
1789bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1790 int64_t &Offset1,
1791 int64_t &Offset2) const {
1792 // Don't worry about Thumb: just ARM and Thumb2.
1793 if (Subtarget.isThumb1Only()) return false;
1794
1795 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1796 return false;
1797
1798 auto IsLoadOpcode = [&](unsigned Opcode) {
1799 switch (Opcode) {
1800 default:
1801 return false;
1802 case ARM::LDRi12:
1803 case ARM::LDRBi12:
1804 case ARM::LDRD:
1805 case ARM::LDRH:
1806 case ARM::LDRSB:
1807 case ARM::LDRSH:
1808 case ARM::VLDRD:
1809 case ARM::VLDRS:
1810 case ARM::t2LDRi8:
1811 case ARM::t2LDRBi8:
1812 case ARM::t2LDRDi8:
1813 case ARM::t2LDRSHi8:
1814 case ARM::t2LDRi12:
1815 case ARM::t2LDRBi12:
1816 case ARM::t2LDRSHi12:
1817 return true;
1818 }
1819 };
1820
1821 if (!IsLoadOpcode(Load1->getMachineOpcode()) ||
1822 !IsLoadOpcode(Load2->getMachineOpcode()))
1823 return false;
1824
1825 // Check if base addresses and chain operands match.
1826 if (Load1->getOperand(Num: 0) != Load2->getOperand(Num: 0) ||
1827 Load1->getOperand(Num: 4) != Load2->getOperand(Num: 4))
1828 return false;
1829
1830 // Index should be Reg0.
1831 if (Load1->getOperand(Num: 3) != Load2->getOperand(Num: 3))
1832 return false;
1833
1834 // Determine the offsets.
1835 if (isa<ConstantSDNode>(Val: Load1->getOperand(Num: 1)) &&
1836 isa<ConstantSDNode>(Val: Load2->getOperand(Num: 1))) {
1837 Offset1 = cast<ConstantSDNode>(Val: Load1->getOperand(Num: 1))->getSExtValue();
1838 Offset2 = cast<ConstantSDNode>(Val: Load2->getOperand(Num: 1))->getSExtValue();
1839 return true;
1840 }
1841
1842 return false;
1843}
1844
1845/// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1846/// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1847/// be scheduled together. On some targets if two loads are loading from
1848/// addresses in the same cache line, it's better if they are scheduled
1849/// together. This function takes two integers that represent the load offsets
1850/// from the common base address. It returns true if it decides it's desirable
1851/// to schedule the two loads together. "NumLoads" is the number of loads that
1852/// have already been scheduled after Load1.
1853///
1854/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1855/// is permanently disabled.
1856bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1857 int64_t Offset1, int64_t Offset2,
1858 unsigned NumLoads) const {
1859 // Don't worry about Thumb: just ARM and Thumb2.
1860 if (Subtarget.isThumb1Only()) return false;
1861
1862 assert(Offset2 > Offset1);
1863
1864 if ((Offset2 - Offset1) / 8 > 64)
1865 return false;
1866
1867 // Check if the machine opcodes are different. If they are different
1868 // then we consider them to not be of the same base address,
1869 // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
1870 // In this case, they are considered to be the same because they are different
1871 // encoding forms of the same basic instruction.
1872 if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
1873 !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
1874 Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
1875 (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
1876 Load2->getMachineOpcode() == ARM::t2LDRBi8)))
1877 return false; // FIXME: overly conservative?
1878
1879 // Four loads in a row should be sufficient.
1880 if (NumLoads >= 3)
1881 return false;
1882
1883 return true;
1884}
1885
1886bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1887 const MachineBasicBlock *MBB,
1888 const MachineFunction &MF) const {
1889 // Debug info is never a scheduling boundary. It's necessary to be explicit
1890 // due to the special treatment of IT instructions below, otherwise a
1891 // dbg_value followed by an IT will result in the IT instruction being
1892 // considered a scheduling hazard, which is wrong. It should be the actual
1893 // instruction preceding the dbg_value instruction(s), just like it is
1894 // when debug info is not present.
1895 if (MI.isDebugInstr())
1896 return false;
1897
1898 // Terminators and labels can't be scheduled around.
1899 if (MI.isTerminator() || MI.isPosition())
1900 return true;
1901
1902 // INLINEASM_BR can jump to another block
1903 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1904 return true;
1905
1906 if (isSEHInstruction(MI))
1907 return true;
1908
1909 // Treat the start of the IT block as a scheduling boundary, but schedule
1910 // t2IT along with all instructions following it.
1911 // FIXME: This is a big hammer. But the alternative is to add all potential
1912 // true and anti dependencies to IT block instructions as implicit operands
1913 // to the t2IT instruction. The added compile time and complexity does not
1914 // seem worth it.
1915 MachineBasicBlock::const_iterator I = MI;
1916 // Make sure to skip any debug instructions
1917 while (++I != MBB->end() && I->isDebugInstr())
1918 ;
1919 if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
1920 return true;
1921
1922 // Don't attempt to schedule around any instruction that defines
1923 // a stack-oriented pointer, as it's unlikely to be profitable. This
1924 // saves compile time, because it doesn't require every single
1925 // stack slot reference to depend on the instruction that does the
1926 // modification.
1927 // Calls don't actually change the stack pointer, even if they have imp-defs.
1928 // No ARM calling conventions change the stack pointer. (X86 calling
1929 // conventions sometimes do).
1930 if (!MI.isCall() && MI.definesRegister(Reg: ARM::SP, /*TRI=*/nullptr))
1931 return true;
1932
1933 return false;
1934}
1935
1936bool ARMBaseInstrInfo::
1937isProfitableToIfCvt(MachineBasicBlock &MBB,
1938 unsigned NumCycles, unsigned ExtraPredCycles,
1939 BranchProbability Probability) const {
1940 if (!NumCycles)
1941 return false;
1942
1943 // If we are optimizing for size, see if the branch in the predecessor can be
1944 // lowered to cbn?z by the constant island lowering pass, and return false if
1945 // so. This results in a shorter instruction sequence.
1946 if (MBB.getParent()->getFunction().hasOptSize()) {
1947 MachineBasicBlock *Pred = *MBB.pred_begin();
1948 if (!Pred->empty()) {
1949 MachineInstr *LastMI = &*Pred->rbegin();
1950 if (LastMI->getOpcode() == ARM::t2Bcc) {
1951 const TargetRegisterInfo *TRI = &getRegisterInfo();
1952 MachineInstr *CmpMI = findCMPToFoldIntoCBZ(Br: LastMI, TRI);
1953 if (CmpMI)
1954 return false;
1955 }
1956 }
1957 }
1958 return isProfitableToIfCvt(TMBB&: MBB, NumT: NumCycles, ExtraT: ExtraPredCycles,
1959 FMBB&: MBB, NumF: 0, ExtraF: 0, Probability);
1960}
1961
1962bool ARMBaseInstrInfo::
1963isProfitableToIfCvt(MachineBasicBlock &TBB,
1964 unsigned TCycles, unsigned TExtra,
1965 MachineBasicBlock &FBB,
1966 unsigned FCycles, unsigned FExtra,
1967 BranchProbability Probability) const {
1968 if (!TCycles)
1969 return false;
1970
1971 // In thumb code we often end up trading one branch for a IT block, and
1972 // if we are cloning the instruction can increase code size. Prevent
1973 // blocks with multiple predecessors from being ifcvted to prevent this
1974 // cloning.
1975 if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
1976 if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
1977 return false;
1978 }
1979
1980 // Attempt to estimate the relative costs of predication versus branching.
1981 // Here we scale up each component of UnpredCost to avoid precision issue when
1982 // scaling TCycles/FCycles by Probability.
1983 const unsigned ScalingUpFactor = 1024;
1984
1985 unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
1986 unsigned UnpredCost;
1987 if (!Subtarget.hasBranchPredictor()) {
1988 // When we don't have a branch predictor it's always cheaper to not take a
1989 // branch than take it, so we have to take that into account.
1990 unsigned NotTakenBranchCost = 1;
1991 unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
1992 unsigned TUnpredCycles, FUnpredCycles;
1993 if (!FCycles) {
1994 // Triangle: TBB is the fallthrough
1995 TUnpredCycles = TCycles + NotTakenBranchCost;
1996 FUnpredCycles = TakenBranchCost;
1997 } else {
1998 // Diamond: TBB is the block that is branched to, FBB is the fallthrough
1999 TUnpredCycles = TCycles + TakenBranchCost;
2000 FUnpredCycles = FCycles + NotTakenBranchCost;
2001 // The branch at the end of FBB will disappear when it's predicated, so
2002 // discount it from PredCost.
2003 PredCost -= 1 * ScalingUpFactor;
2004 }
2005 // The total cost is the cost of each path scaled by their probabilities
2006 unsigned TUnpredCost = Probability.scale(Num: TUnpredCycles * ScalingUpFactor);
2007 unsigned FUnpredCost = Probability.getCompl().scale(Num: FUnpredCycles * ScalingUpFactor);
2008 UnpredCost = TUnpredCost + FUnpredCost;
2009 // When predicating assume that the first IT can be folded away but later
2010 // ones cost one cycle each
2011 if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2012 PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2013 }
2014 } else {
2015 unsigned TUnpredCost = Probability.scale(Num: TCycles * ScalingUpFactor);
2016 unsigned FUnpredCost =
2017 Probability.getCompl().scale(Num: FCycles * ScalingUpFactor);
2018 UnpredCost = TUnpredCost + FUnpredCost;
2019 UnpredCost += 1 * ScalingUpFactor; // The branch itself
2020 UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2021 }
2022
2023 return PredCost <= UnpredCost;
2024}
2025
2026unsigned
2027ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2028 unsigned NumInsts) const {
2029 // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2030 // ARM has a condition code field in every predicable instruction, using it
2031 // doesn't change code size.
2032 if (!Subtarget.isThumb2())
2033 return 0;
2034
2035 // It's possible that the size of the IT is restricted to a single block.
2036 unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4;
2037 return divideCeil(Numerator: NumInsts, Denominator: MaxInsts) * 2;
2038}
2039
2040unsigned
2041ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2042 // If this branch is likely to be folded into the comparison to form a
2043 // CB(N)Z, then removing it won't reduce code size at all, because that will
2044 // just replace the CB(N)Z with a CMP.
2045 if (MI.getOpcode() == ARM::t2Bcc &&
2046 findCMPToFoldIntoCBZ(Br: &MI, TRI: &getRegisterInfo()))
2047 return 0;
2048
2049 unsigned Size = getInstSizeInBytes(MI);
2050
2051 // For Thumb2, all branches are 32-bit instructions during the if conversion
2052 // pass, but may be replaced with 16-bit instructions during size reduction.
2053 // Since the branches considered by if conversion tend to be forward branches
2054 // over small basic blocks, they are very likely to be in range for the
2055 // narrow instructions, so we assume the final code size will be half what it
2056 // currently is.
2057 if (Subtarget.isThumb2())
2058 Size /= 2;
2059
2060 return Size;
2061}
2062
2063bool
2064ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2065 MachineBasicBlock &FMBB) const {
2066 // Reduce false anti-dependencies to let the target's out-of-order execution
2067 // engine do its thing.
2068 return Subtarget.isProfitableToUnpredicate();
2069}
2070
2071/// getInstrPredicate - If instruction is predicated, returns its predicate
2072/// condition, otherwise returns AL. It also returns the condition code
2073/// register by reference.
2074ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2075 Register &PredReg) {
2076 int PIdx = MI.findFirstPredOperandIdx();
2077 if (PIdx == -1) {
2078 PredReg = 0;
2079 return ARMCC::AL;
2080 }
2081
2082 PredReg = MI.getOperand(i: PIdx+1).getReg();
2083 return (ARMCC::CondCodes)MI.getOperand(i: PIdx).getImm();
2084}
2085
2086unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2087 if (Opc == ARM::B)
2088 return ARM::Bcc;
2089 if (Opc == ARM::tB)
2090 return ARM::tBcc;
2091 if (Opc == ARM::t2B)
2092 return ARM::t2Bcc;
2093
2094 llvm_unreachable("Unknown unconditional branch opcode!");
2095}
2096
2097MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2098 bool NewMI,
2099 unsigned OpIdx1,
2100 unsigned OpIdx2) const {
2101 switch (MI.getOpcode()) {
2102 case ARM::MOVCCr:
2103 case ARM::t2MOVCCr: {
2104 // MOVCC can be commuted by inverting the condition.
2105 Register PredReg;
2106 ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2107 // MOVCC AL can't be inverted. Shouldn't happen.
2108 if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2109 return nullptr;
2110 MachineInstr *CommutedMI =
2111 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2112 if (!CommutedMI)
2113 return nullptr;
2114 // After swapping the MOVCC operands, also invert the condition.
2115 CommutedMI->getOperand(i: CommutedMI->findFirstPredOperandIdx())
2116 .setImm(ARMCC::getOppositeCondition(CC));
2117 return CommutedMI;
2118 }
2119 }
2120 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2121}
2122
2123/// Identify instructions that can be folded into a MOVCC instruction, and
2124/// return the defining instruction.
2125MachineInstr *
2126ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2127 const TargetInstrInfo *TII) const {
2128 if (!Reg.isVirtual())
2129 return nullptr;
2130 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2131 return nullptr;
2132 MachineInstr *MI = MRI.getVRegDef(Reg);
2133 if (!MI)
2134 return nullptr;
2135 // Check if MI can be predicated and folded into the MOVCC.
2136 if (!isPredicable(MI: *MI))
2137 return nullptr;
2138 // Check if MI has any non-dead defs or physreg uses. This also detects
2139 // predicated instructions which will be reading CPSR.
2140 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 1)) {
2141 // Reject frame index operands, PEI can't handle the predicated pseudos.
2142 if (MO.isFI() || MO.isCPI() || MO.isJTI())
2143 return nullptr;
2144 if (!MO.isReg())
2145 continue;
2146 // MI can't have any tied operands, that would conflict with predication.
2147 if (MO.isTied())
2148 return nullptr;
2149 if (MO.getReg().isPhysical())
2150 return nullptr;
2151 if (MO.isDef() && !MO.isDead())
2152 return nullptr;
2153 }
2154 bool DontMoveAcrossStores = true;
2155 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
2156 return nullptr;
2157 return MI;
2158}
2159
2160MachineInstr *
2161ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2162 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2163 bool PreferFalse) const {
2164 assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2165 "Unknown select instruction");
2166 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2167 MachineInstr *DefMI = canFoldIntoMOVCC(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this);
2168 bool Invert = !DefMI;
2169 if (!DefMI)
2170 DefMI = canFoldIntoMOVCC(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this);
2171 if (!DefMI)
2172 return nullptr;
2173
2174 // Find new register class to use.
2175 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
2176 MachineOperand TrueReg = MI.getOperand(i: Invert ? 1 : 2);
2177 Register DestReg = MI.getOperand(i: 0).getReg();
2178 const TargetRegisterClass *FalseClass = MRI.getRegClass(Reg: FalseReg.getReg());
2179 const TargetRegisterClass *TrueClass = MRI.getRegClass(Reg: TrueReg.getReg());
2180 if (!MRI.constrainRegClass(Reg: DestReg, RC: FalseClass))
2181 return nullptr;
2182 if (!MRI.constrainRegClass(Reg: DestReg, RC: TrueClass))
2183 return nullptr;
2184
2185 // Create a new predicated version of DefMI.
2186 // Rfalse is the first use.
2187 MachineInstrBuilder NewMI =
2188 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: DefMI->getDesc(), DestReg);
2189
2190 // Copy all the DefMI operands, excluding its (null) predicate.
2191 const MCInstrDesc &DefDesc = DefMI->getDesc();
2192 for (unsigned i = 1, e = DefDesc.getNumOperands();
2193 i != e && !DefDesc.operands()[i].isPredicate(); ++i)
2194 NewMI.add(MO: DefMI->getOperand(i));
2195
2196 unsigned CondCode = MI.getOperand(i: 3).getImm();
2197 if (Invert)
2198 NewMI.addImm(Val: ARMCC::getOppositeCondition(CC: ARMCC::CondCodes(CondCode)));
2199 else
2200 NewMI.addImm(Val: CondCode);
2201 NewMI.add(MO: MI.getOperand(i: 4));
2202
2203 // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2204 if (NewMI->hasOptionalDef())
2205 NewMI.add(MO: condCodeOp());
2206
2207 // The output register value when the predicate is false is an implicit
2208 // register operand tied to the first def.
2209 // The tie makes the register allocator ensure the FalseReg is allocated the
2210 // same register as operand 0.
2211 FalseReg.setImplicit();
2212 NewMI.add(MO: FalseReg);
2213 NewMI->tieOperands(DefIdx: 0, UseIdx: NewMI->getNumOperands() - 1);
2214
2215 // Update SeenMIs set: register newly created MI and erase removed DefMI.
2216 SeenMIs.insert(Ptr: NewMI);
2217 SeenMIs.erase(Ptr: DefMI);
2218
2219 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2220 // DefMI would be invalid when transferred inside the loop. Checking for a
2221 // loop is expensive, but at least remove kill flags if they are in different
2222 // BBs.
2223 if (DefMI->getParent() != MI.getParent())
2224 NewMI->clearKillInfo();
2225
2226 // The caller will erase MI, but not DefMI.
2227 DefMI->eraseFromParent();
2228 return NewMI;
2229}
2230
2231/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2232/// instruction is encoded with an 'S' bit is determined by the optional CPSR
2233/// def operand.
2234///
2235/// This will go away once we can teach tblgen how to set the optional CPSR def
2236/// operand itself.
2237struct AddSubFlagsOpcodePair {
2238 uint16_t PseudoOpc;
2239 uint16_t MachineOpc;
2240};
2241
2242static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2243 {.PseudoOpc: ARM::ADDSri, .MachineOpc: ARM::ADDri},
2244 {.PseudoOpc: ARM::ADDSrr, .MachineOpc: ARM::ADDrr},
2245 {.PseudoOpc: ARM::ADDSrsi, .MachineOpc: ARM::ADDrsi},
2246 {.PseudoOpc: ARM::ADDSrsr, .MachineOpc: ARM::ADDrsr},
2247
2248 {.PseudoOpc: ARM::SUBSri, .MachineOpc: ARM::SUBri},
2249 {.PseudoOpc: ARM::SUBSrr, .MachineOpc: ARM::SUBrr},
2250 {.PseudoOpc: ARM::SUBSrsi, .MachineOpc: ARM::SUBrsi},
2251 {.PseudoOpc: ARM::SUBSrsr, .MachineOpc: ARM::SUBrsr},
2252
2253 {.PseudoOpc: ARM::RSBSri, .MachineOpc: ARM::RSBri},
2254 {.PseudoOpc: ARM::RSBSrsi, .MachineOpc: ARM::RSBrsi},
2255 {.PseudoOpc: ARM::RSBSrsr, .MachineOpc: ARM::RSBrsr},
2256
2257 {.PseudoOpc: ARM::tADDSi3, .MachineOpc: ARM::tADDi3},
2258 {.PseudoOpc: ARM::tADDSi8, .MachineOpc: ARM::tADDi8},
2259 {.PseudoOpc: ARM::tADDSrr, .MachineOpc: ARM::tADDrr},
2260 {.PseudoOpc: ARM::tADCS, .MachineOpc: ARM::tADC},
2261
2262 {.PseudoOpc: ARM::tSUBSi3, .MachineOpc: ARM::tSUBi3},
2263 {.PseudoOpc: ARM::tSUBSi8, .MachineOpc: ARM::tSUBi8},
2264 {.PseudoOpc: ARM::tSUBSrr, .MachineOpc: ARM::tSUBrr},
2265 {.PseudoOpc: ARM::tSBCS, .MachineOpc: ARM::tSBC},
2266 {.PseudoOpc: ARM::tRSBS, .MachineOpc: ARM::tRSB},
2267 {.PseudoOpc: ARM::tLSLSri, .MachineOpc: ARM::tLSLri},
2268
2269 {.PseudoOpc: ARM::t2ADDSri, .MachineOpc: ARM::t2ADDri},
2270 {.PseudoOpc: ARM::t2ADDSrr, .MachineOpc: ARM::t2ADDrr},
2271 {.PseudoOpc: ARM::t2ADDSrs, .MachineOpc: ARM::t2ADDrs},
2272
2273 {.PseudoOpc: ARM::t2SUBSri, .MachineOpc: ARM::t2SUBri},
2274 {.PseudoOpc: ARM::t2SUBSrr, .MachineOpc: ARM::t2SUBrr},
2275 {.PseudoOpc: ARM::t2SUBSrs, .MachineOpc: ARM::t2SUBrs},
2276
2277 {.PseudoOpc: ARM::t2RSBSri, .MachineOpc: ARM::t2RSBri},
2278 {.PseudoOpc: ARM::t2RSBSrs, .MachineOpc: ARM::t2RSBrs},
2279};
2280
2281unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2282 for (const auto &Entry : AddSubFlagsOpcodeMap)
2283 if (OldOpc == Entry.PseudoOpc)
2284 return Entry.MachineOpc;
2285 return 0;
2286}
2287
2288void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2289 MachineBasicBlock::iterator &MBBI,
2290 const DebugLoc &dl, Register DestReg,
2291 Register BaseReg, int NumBytes,
2292 ARMCC::CondCodes Pred, Register PredReg,
2293 const ARMBaseInstrInfo &TII,
2294 unsigned MIFlags) {
2295 if (NumBytes == 0 && DestReg != BaseReg) {
2296 BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::MOVr), DestReg)
2297 .addReg(RegNo: BaseReg, Flags: RegState::Kill)
2298 .add(MOs: predOps(Pred, PredReg))
2299 .add(MO: condCodeOp())
2300 .setMIFlags(MIFlags);
2301 return;
2302 }
2303
2304 bool isSub = NumBytes < 0;
2305 if (isSub) NumBytes = -NumBytes;
2306
2307 while (NumBytes) {
2308 unsigned RotAmt = ARM_AM::getSOImmValRotate(Imm: NumBytes);
2309 unsigned ThisVal = NumBytes & llvm::rotr<uint32_t>(V: 0xFF, R: RotAmt);
2310 assert(ThisVal && "Didn't extract field correctly");
2311
2312 // We will handle these bits from offset, clear them.
2313 NumBytes &= ~ThisVal;
2314
2315 assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2316
2317 // Build the new ADD / SUB.
2318 unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2319 BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: Opc), DestReg)
2320 .addReg(RegNo: BaseReg, Flags: RegState::Kill)
2321 .addImm(Val: ThisVal)
2322 .add(MOs: predOps(Pred, PredReg))
2323 .add(MO: condCodeOp())
2324 .setMIFlags(MIFlags);
2325 BaseReg = DestReg;
2326 }
2327}
2328
2329bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2330 MachineFunction &MF, MachineInstr *MI,
2331 unsigned NumBytes) {
2332 // This optimisation potentially adds lots of load and store
2333 // micro-operations, it's only really a great benefit to code-size.
2334 if (!Subtarget.hasMinSize())
2335 return false;
2336
2337 // If only one register is pushed/popped, LLVM can use an LDR/STR
2338 // instead. We can't modify those so make sure we're dealing with an
2339 // instruction we understand.
2340 bool IsPop = isPopOpcode(Opc: MI->getOpcode());
2341 bool IsPush = isPushOpcode(Opc: MI->getOpcode());
2342 if (!IsPush && !IsPop)
2343 return false;
2344
2345 bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2346 MI->getOpcode() == ARM::VLDMDIA_UPD;
2347 bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2348 MI->getOpcode() == ARM::tPOP ||
2349 MI->getOpcode() == ARM::tPOP_RET;
2350
2351 assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2352 MI->getOperand(1).getReg() == ARM::SP)) &&
2353 "trying to fold sp update into non-sp-updating push/pop");
2354
2355 // The VFP push & pop act on D-registers, so we can only fold an adjustment
2356 // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2357 // if this is violated.
2358 if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2359 return false;
2360
2361 // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2362 // pred) so the list starts at 4. Thumb1 starts after the predicate.
2363 int RegListIdx = IsT1PushPop ? 2 : 4;
2364
2365 // Calculate the space we'll need in terms of registers.
2366 unsigned RegsNeeded;
2367 const TargetRegisterClass *RegClass;
2368 if (IsVFPPushPop) {
2369 RegsNeeded = NumBytes / 8;
2370 RegClass = &ARM::DPRRegClass;
2371 } else {
2372 RegsNeeded = NumBytes / 4;
2373 RegClass = &ARM::GPRRegClass;
2374 }
2375
2376 // We're going to have to strip all list operands off before
2377 // re-adding them since the order matters, so save the existing ones
2378 // for later.
2379 SmallVector<MachineOperand, 4> RegList;
2380
2381 // We're also going to need the first register transferred by this
2382 // instruction, which won't necessarily be the first register in the list.
2383 unsigned FirstRegEnc = -1;
2384
2385 const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2386 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2387 MachineOperand &MO = MI->getOperand(i);
2388 RegList.push_back(Elt: MO);
2389
2390 if (MO.isReg() && !MO.isImplicit() &&
2391 TRI->getEncodingValue(Reg: MO.getReg()) < FirstRegEnc)
2392 FirstRegEnc = TRI->getEncodingValue(Reg: MO.getReg());
2393 }
2394
2395 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(MF: &MF);
2396
2397 // Now try to find enough space in the reglist to allocate NumBytes.
2398 for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2399 --CurRegEnc) {
2400 MCRegister CurReg = RegClass->getRegister(i: CurRegEnc);
2401 if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(Reg: ARM::R7))
2402 continue;
2403 if (!IsPop) {
2404 // Pushing any register is completely harmless, mark the register involved
2405 // as undef since we don't care about its value and must not restore it
2406 // during stack unwinding.
2407 RegList.push_back(Elt: MachineOperand::CreateReg(Reg: CurReg, isDef: false, isImp: false,
2408 isKill: false, isDead: false, isUndef: true));
2409 --RegsNeeded;
2410 continue;
2411 }
2412
2413 // However, we can only pop an extra register if it's not live. For
2414 // registers live within the function we might clobber a return value
2415 // register; the other way a register can be live here is if it's
2416 // callee-saved.
2417 if (isCalleeSavedRegister(Reg: CurReg, CSRegs) ||
2418 MI->getParent()->computeRegisterLiveness(TRI, Reg: CurReg, Before: MI) !=
2419 MachineBasicBlock::LQR_Dead) {
2420 // VFP pops don't allow holes in the register list, so any skip is fatal
2421 // for our transformation. GPR pops do, so we should just keep looking.
2422 if (IsVFPPushPop)
2423 return false;
2424 else
2425 continue;
2426 }
2427
2428 // Mark the unimportant registers as <def,dead> in the POP.
2429 RegList.push_back(Elt: MachineOperand::CreateReg(Reg: CurReg, isDef: true, isImp: false, isKill: false,
2430 isDead: true));
2431 --RegsNeeded;
2432 }
2433
2434 if (RegsNeeded > 0)
2435 return false;
2436
2437 // Finally we know we can profitably perform the optimisation so go
2438 // ahead: strip all existing registers off and add them back again
2439 // in the right order.
2440 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2441 MI->removeOperand(OpNo: i);
2442
2443 // Add the complete list back in.
2444 MachineInstrBuilder MIB(MF, &*MI);
2445 for (const MachineOperand &MO : llvm::reverse(C&: RegList))
2446 MIB.add(MO);
2447
2448 return true;
2449}
2450
2451bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2452 Register FrameReg, int &Offset,
2453 const ARMBaseInstrInfo &TII) {
2454 unsigned Opcode = MI.getOpcode();
2455 const MCInstrDesc &Desc = MI.getDesc();
2456 unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2457 bool isSub = false;
2458
2459 // Memory operands in inline assembly always use AddrMode2.
2460 if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2461 AddrMode = ARMII::AddrMode2;
2462
2463 if (Opcode == ARM::ADDri) {
2464 Offset += MI.getOperand(i: FrameRegIdx+1).getImm();
2465 if (Offset == 0) {
2466 // Turn it into a move.
2467 MI.setDesc(TII.get(Opcode: ARM::MOVr));
2468 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2469 MI.removeOperand(OpNo: FrameRegIdx+1);
2470 Offset = 0;
2471 return true;
2472 } else if (Offset < 0) {
2473 Offset = -Offset;
2474 isSub = true;
2475 MI.setDesc(TII.get(Opcode: ARM::SUBri));
2476 }
2477
2478 // Common case: small offset, fits into instruction.
2479 if (ARM_AM::getSOImmVal(Arg: Offset) != -1) {
2480 // Replace the FrameIndex with sp / fp
2481 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2482 MI.getOperand(i: FrameRegIdx+1).ChangeToImmediate(ImmVal: Offset);
2483 Offset = 0;
2484 return true;
2485 }
2486
2487 // Otherwise, pull as much of the immediate into this ADDri/SUBri
2488 // as possible.
2489 unsigned RotAmt = ARM_AM::getSOImmValRotate(Imm: Offset);
2490 unsigned ThisImmVal = Offset & llvm::rotr<uint32_t>(V: 0xFF, R: RotAmt);
2491
2492 // We will handle these bits from offset, clear them.
2493 Offset &= ~ThisImmVal;
2494
2495 // Get the properly encoded SOImmVal field.
2496 assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2497 "Bit extraction didn't work?");
2498 MI.getOperand(i: FrameRegIdx+1).ChangeToImmediate(ImmVal: ThisImmVal);
2499 } else {
2500 unsigned ImmIdx = 0;
2501 int InstrOffs = 0;
2502 unsigned NumBits = 0;
2503 unsigned Scale = 1;
2504 switch (AddrMode) {
2505 case ARMII::AddrMode_i12:
2506 ImmIdx = FrameRegIdx + 1;
2507 InstrOffs = MI.getOperand(i: ImmIdx).getImm();
2508 NumBits = 12;
2509 break;
2510 case ARMII::AddrMode2:
2511 ImmIdx = FrameRegIdx+2;
2512 InstrOffs = ARM_AM::getAM2Offset(AM2Opc: MI.getOperand(i: ImmIdx).getImm());
2513 if (ARM_AM::getAM2Op(AM2Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2514 InstrOffs *= -1;
2515 NumBits = 12;
2516 break;
2517 case ARMII::AddrMode3:
2518 ImmIdx = FrameRegIdx+2;
2519 InstrOffs = ARM_AM::getAM3Offset(AM3Opc: MI.getOperand(i: ImmIdx).getImm());
2520 if (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2521 InstrOffs *= -1;
2522 NumBits = 8;
2523 break;
2524 case ARMII::AddrMode4:
2525 case ARMII::AddrMode6:
2526 // Can't fold any offset even if it's zero.
2527 return false;
2528 case ARMII::AddrMode5:
2529 ImmIdx = FrameRegIdx+1;
2530 InstrOffs = ARM_AM::getAM5Offset(AM5Opc: MI.getOperand(i: ImmIdx).getImm());
2531 if (ARM_AM::getAM5Op(AM5Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2532 InstrOffs *= -1;
2533 NumBits = 8;
2534 Scale = 4;
2535 break;
2536 case ARMII::AddrMode5FP16:
2537 ImmIdx = FrameRegIdx+1;
2538 InstrOffs = ARM_AM::getAM5Offset(AM5Opc: MI.getOperand(i: ImmIdx).getImm());
2539 if (ARM_AM::getAM5Op(AM5Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2540 InstrOffs *= -1;
2541 NumBits = 8;
2542 Scale = 2;
2543 break;
2544 case ARMII::AddrModeT2_i7:
2545 case ARMII::AddrModeT2_i7s2:
2546 case ARMII::AddrModeT2_i7s4:
2547 ImmIdx = FrameRegIdx+1;
2548 InstrOffs = MI.getOperand(i: ImmIdx).getImm();
2549 NumBits = 7;
2550 Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2551 AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2552 break;
2553 default:
2554 llvm_unreachable("Unsupported addressing mode!");
2555 }
2556
2557 Offset += InstrOffs * Scale;
2558 assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2559 if (Offset < 0) {
2560 Offset = -Offset;
2561 isSub = true;
2562 }
2563
2564 // Attempt to fold address comp. if opcode has offset bits
2565 if (NumBits > 0) {
2566 // Common case: small offset, fits into instruction.
2567 MachineOperand &ImmOp = MI.getOperand(i: ImmIdx);
2568 int ImmedOffset = Offset / Scale;
2569 unsigned Mask = (1 << NumBits) - 1;
2570 if ((unsigned)Offset <= Mask * Scale) {
2571 // Replace the FrameIndex with sp
2572 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2573 // FIXME: When addrmode2 goes away, this will simplify (like the
2574 // T2 version), as the LDR.i12 versions don't need the encoding
2575 // tricks for the offset value.
2576 if (isSub) {
2577 if (AddrMode == ARMII::AddrMode_i12)
2578 ImmedOffset = -ImmedOffset;
2579 else
2580 ImmedOffset |= 1 << NumBits;
2581 }
2582 ImmOp.ChangeToImmediate(ImmVal: ImmedOffset);
2583 Offset = 0;
2584 return true;
2585 }
2586
2587 // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2588 ImmedOffset = ImmedOffset & Mask;
2589 if (isSub) {
2590 if (AddrMode == ARMII::AddrMode_i12)
2591 ImmedOffset = -ImmedOffset;
2592 else
2593 ImmedOffset |= 1 << NumBits;
2594 }
2595 ImmOp.ChangeToImmediate(ImmVal: ImmedOffset);
2596 Offset &= ~(Mask*Scale);
2597 }
2598 }
2599
2600 Offset = (isSub) ? -Offset : Offset;
2601 return Offset == 0;
2602}
2603
2604/// analyzeCompare - For a comparison instruction, return the source registers
2605/// in SrcReg and SrcReg2 if having two register operands, and the value it
2606/// compares against in CmpValue. Return true if the comparison instruction
2607/// can be analyzed.
2608bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2609 Register &SrcReg2, int64_t &CmpMask,
2610 int64_t &CmpValue) const {
2611 switch (MI.getOpcode()) {
2612 default: break;
2613 case ARM::CMPri:
2614 case ARM::t2CMPri:
2615 case ARM::tCMPi8:
2616 SrcReg = MI.getOperand(i: 0).getReg();
2617 SrcReg2 = 0;
2618 CmpMask = ~0;
2619 CmpValue = MI.getOperand(i: 1).getImm();
2620 return true;
2621 case ARM::CMPrr:
2622 case ARM::t2CMPrr:
2623 case ARM::tCMPr:
2624 SrcReg = MI.getOperand(i: 0).getReg();
2625 SrcReg2 = MI.getOperand(i: 1).getReg();
2626 CmpMask = ~0;
2627 CmpValue = 0;
2628 return true;
2629 case ARM::TSTri:
2630 case ARM::t2TSTri:
2631 SrcReg = MI.getOperand(i: 0).getReg();
2632 SrcReg2 = 0;
2633 CmpMask = MI.getOperand(i: 1).getImm();
2634 CmpValue = 0;
2635 return true;
2636 }
2637
2638 return false;
2639}
2640
2641/// isSuitableForMask - Identify a suitable 'and' instruction that
2642/// operates on the given source register and applies the same mask
2643/// as a 'tst' instruction. Provide a limited look-through for copies.
2644/// When successful, MI will hold the found instruction.
2645static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2646 int CmpMask, bool CommonUse) {
2647 switch (MI->getOpcode()) {
2648 case ARM::ANDri:
2649 case ARM::t2ANDri:
2650 if (CmpMask != MI->getOperand(i: 2).getImm())
2651 return false;
2652 if (SrcReg == MI->getOperand(i: CommonUse ? 1 : 0).getReg())
2653 return true;
2654 break;
2655 }
2656
2657 return false;
2658}
2659
2660/// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2661/// the condition code if we modify the instructions such that flags are
2662/// set by ADD(a,b,X).
2663inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2664 switch (CC) {
2665 default: return ARMCC::AL;
2666 case ARMCC::HS: return ARMCC::LO;
2667 case ARMCC::LO: return ARMCC::HS;
2668 case ARMCC::VS: return ARMCC::VS;
2669 case ARMCC::VC: return ARMCC::VC;
2670 }
2671}
2672
2673/// isRedundantFlagInstr - check whether the first instruction, whose only
2674/// purpose is to update flags, can be made redundant.
2675/// CMPrr can be made redundant by SUBrr if the operands are the same.
2676/// CMPri can be made redundant by SUBri if the operands are the same.
2677/// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2678/// This function can be extended later on.
2679inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2680 Register SrcReg, Register SrcReg2,
2681 int64_t ImmValue,
2682 const MachineInstr *OI,
2683 bool &IsThumb1) {
2684 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2685 (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2686 ((OI->getOperand(i: 1).getReg() == SrcReg &&
2687 OI->getOperand(i: 2).getReg() == SrcReg2) ||
2688 (OI->getOperand(i: 1).getReg() == SrcReg2 &&
2689 OI->getOperand(i: 2).getReg() == SrcReg))) {
2690 IsThumb1 = false;
2691 return true;
2692 }
2693
2694 if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2695 ((OI->getOperand(i: 2).getReg() == SrcReg &&
2696 OI->getOperand(i: 3).getReg() == SrcReg2) ||
2697 (OI->getOperand(i: 2).getReg() == SrcReg2 &&
2698 OI->getOperand(i: 3).getReg() == SrcReg))) {
2699 IsThumb1 = true;
2700 return true;
2701 }
2702
2703 if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2704 (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2705 OI->getOperand(i: 1).getReg() == SrcReg &&
2706 OI->getOperand(i: 2).getImm() == ImmValue) {
2707 IsThumb1 = false;
2708 return true;
2709 }
2710
2711 if (CmpI->getOpcode() == ARM::tCMPi8 &&
2712 (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2713 OI->getOperand(i: 2).getReg() == SrcReg &&
2714 OI->getOperand(i: 3).getImm() == ImmValue) {
2715 IsThumb1 = true;
2716 return true;
2717 }
2718
2719 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2720 (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2721 OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2722 OI->getOperand(i: 0).isReg() && OI->getOperand(i: 1).isReg() &&
2723 OI->getOperand(i: 0).getReg() == SrcReg &&
2724 OI->getOperand(i: 1).getReg() == SrcReg2) {
2725 IsThumb1 = false;
2726 return true;
2727 }
2728
2729 if (CmpI->getOpcode() == ARM::tCMPr &&
2730 (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2731 OI->getOpcode() == ARM::tADDrr) &&
2732 OI->getOperand(i: 0).getReg() == SrcReg &&
2733 OI->getOperand(i: 2).getReg() == SrcReg2) {
2734 IsThumb1 = true;
2735 return true;
2736 }
2737
2738 return false;
2739}
2740
2741static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2742 switch (MI->getOpcode()) {
2743 default: return false;
2744 case ARM::tLSLri:
2745 case ARM::tLSRri:
2746 case ARM::tLSLrr:
2747 case ARM::tLSRrr:
2748 case ARM::tSUBrr:
2749 case ARM::tADDrr:
2750 case ARM::tADDi3:
2751 case ARM::tADDi8:
2752 case ARM::tSUBi3:
2753 case ARM::tSUBi8:
2754 case ARM::tMUL:
2755 case ARM::tADC:
2756 case ARM::tSBC:
2757 case ARM::tRSB:
2758 case ARM::tAND:
2759 case ARM::tORR:
2760 case ARM::tEOR:
2761 case ARM::tBIC:
2762 case ARM::tMVN:
2763 case ARM::tASRri:
2764 case ARM::tASRrr:
2765 case ARM::tROR:
2766 IsThumb1 = true;
2767 [[fallthrough]];
2768 case ARM::RSBrr:
2769 case ARM::RSBri:
2770 case ARM::RSCrr:
2771 case ARM::RSCri:
2772 case ARM::ADDrr:
2773 case ARM::ADDri:
2774 case ARM::ADCrr:
2775 case ARM::ADCri:
2776 case ARM::SUBrr:
2777 case ARM::SUBri:
2778 case ARM::SBCrr:
2779 case ARM::SBCri:
2780 case ARM::t2RSBri:
2781 case ARM::t2ADDrr:
2782 case ARM::t2ADDri:
2783 case ARM::t2ADCrr:
2784 case ARM::t2ADCri:
2785 case ARM::t2SUBrr:
2786 case ARM::t2SUBri:
2787 case ARM::t2SBCrr:
2788 case ARM::t2SBCri:
2789 case ARM::ANDrr:
2790 case ARM::ANDri:
2791 case ARM::ANDrsr:
2792 case ARM::ANDrsi:
2793 case ARM::t2ANDrr:
2794 case ARM::t2ANDri:
2795 case ARM::t2ANDrs:
2796 case ARM::ORRrr:
2797 case ARM::ORRri:
2798 case ARM::ORRrsr:
2799 case ARM::ORRrsi:
2800 case ARM::t2ORRrr:
2801 case ARM::t2ORRri:
2802 case ARM::t2ORRrs:
2803 case ARM::EORrr:
2804 case ARM::EORri:
2805 case ARM::EORrsr:
2806 case ARM::EORrsi:
2807 case ARM::t2EORrr:
2808 case ARM::t2EORri:
2809 case ARM::t2EORrs:
2810 case ARM::BICri:
2811 case ARM::BICrr:
2812 case ARM::BICrsi:
2813 case ARM::BICrsr:
2814 case ARM::t2BICri:
2815 case ARM::t2BICrr:
2816 case ARM::t2BICrs:
2817 case ARM::t2LSRri:
2818 case ARM::t2LSRrr:
2819 case ARM::t2LSLri:
2820 case ARM::t2LSLrr:
2821 case ARM::MOVsr:
2822 case ARM::MOVsi:
2823 return true;
2824 }
2825}
2826
2827/// optimizeCompareInstr - Convert the instruction supplying the argument to the
2828/// comparison into one that sets the zero bit in the flags register;
2829/// Remove a redundant Compare instruction if an earlier instruction can set the
2830/// flags in the same way as Compare.
2831/// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2832/// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2833/// condition code of instructions which use the flags.
2834bool ARMBaseInstrInfo::optimizeCompareInstr(
2835 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2836 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2837 // Get the unique definition of SrcReg.
2838 MachineInstr *MI = MRI->getUniqueVRegDef(Reg: SrcReg);
2839 if (!MI) return false;
2840
2841 // Masked compares sometimes use the same register as the corresponding 'and'.
2842 if (CmpMask != ~0) {
2843 if (!isSuitableForMask(MI, SrcReg, CmpMask, CommonUse: false) || isPredicated(MI: *MI)) {
2844 MI = nullptr;
2845 for (MachineRegisterInfo::use_instr_iterator
2846 UI = MRI->use_instr_begin(RegNo: SrcReg), UE = MRI->use_instr_end();
2847 UI != UE; ++UI) {
2848 if (UI->getParent() != CmpInstr.getParent())
2849 continue;
2850 MachineInstr *PotentialAND = &*UI;
2851 if (!isSuitableForMask(MI&: PotentialAND, SrcReg, CmpMask, CommonUse: true) ||
2852 isPredicated(MI: *PotentialAND))
2853 continue;
2854 MI = PotentialAND;
2855 break;
2856 }
2857 if (!MI) return false;
2858 }
2859 }
2860
2861 // Get ready to iterate backward from CmpInstr.
2862 MachineBasicBlock::iterator I = CmpInstr, E = MI,
2863 B = CmpInstr.getParent()->begin();
2864
2865 // Early exit if CmpInstr is at the beginning of the BB.
2866 if (I == B) return false;
2867
2868 // There are two possible candidates which can be changed to set CPSR:
2869 // One is MI, the other is a SUB or ADD instruction.
2870 // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
2871 // ADDr[ri](r1, r2, X).
2872 // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
2873 MachineInstr *SubAdd = nullptr;
2874 if (SrcReg2 != 0)
2875 // MI is not a candidate for CMPrr.
2876 MI = nullptr;
2877 else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
2878 // Conservatively refuse to convert an instruction which isn't in the same
2879 // BB as the comparison.
2880 // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
2881 // Thus we cannot return here.
2882 if (CmpInstr.getOpcode() == ARM::CMPri ||
2883 CmpInstr.getOpcode() == ARM::t2CMPri ||
2884 CmpInstr.getOpcode() == ARM::tCMPi8)
2885 MI = nullptr;
2886 else
2887 return false;
2888 }
2889
2890 bool IsThumb1 = false;
2891 if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
2892 return false;
2893
2894 // We also want to do this peephole for cases like this: if (a*b == 0),
2895 // and optimise away the CMP instruction from the generated code sequence:
2896 // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
2897 // resulting from the select instruction, but these MOVS instructions for
2898 // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
2899 // However, if we only have MOVS instructions in between the CMP and the
2900 // other instruction (the MULS in this example), then the CPSR is dead so we
2901 // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
2902 // reordering and then continue the analysis hoping we can eliminate the
2903 // CMP. This peephole works on the vregs, so is still in SSA form. As a
2904 // consequence, the movs won't redefine/kill the MUL operands which would
2905 // make this reordering illegal.
2906 const TargetRegisterInfo *TRI = &getRegisterInfo();
2907 if (MI && IsThumb1) {
2908 --I;
2909 if (I != E && !MI->readsRegister(Reg: ARM::CPSR, TRI)) {
2910 bool CanReorder = true;
2911 for (; I != E; --I) {
2912 if (I->getOpcode() != ARM::tMOVi8) {
2913 CanReorder = false;
2914 break;
2915 }
2916 }
2917 if (CanReorder) {
2918 MI = MI->removeFromParent();
2919 E = CmpInstr;
2920 CmpInstr.getParent()->insert(I: E, MI);
2921 }
2922 }
2923 I = CmpInstr;
2924 E = MI;
2925 }
2926
2927 // Check that CPSR isn't set between the comparison instruction and the one we
2928 // want to change. At the same time, search for SubAdd.
2929 bool SubAddIsThumb1 = false;
2930 do {
2931 const MachineInstr &Instr = *--I;
2932
2933 // Check whether CmpInstr can be made redundant by the current instruction.
2934 if (isRedundantFlagInstr(CmpI: &CmpInstr, SrcReg, SrcReg2, ImmValue: CmpValue, OI: &Instr,
2935 IsThumb1&: SubAddIsThumb1)) {
2936 SubAdd = &*I;
2937 break;
2938 }
2939
2940 // Allow E (which was initially MI) to be SubAdd but do not search before E.
2941 if (I == E)
2942 break;
2943
2944 if (Instr.modifiesRegister(Reg: ARM::CPSR, TRI) ||
2945 Instr.readsRegister(Reg: ARM::CPSR, TRI))
2946 // This instruction modifies or uses CPSR after the one we want to
2947 // change. We can't do this transformation.
2948 return false;
2949
2950 if (I == B) {
2951 // In some cases, we scan the use-list of an instruction for an AND;
2952 // that AND is in the same BB, but may not be scheduled before the
2953 // corresponding TST. In that case, bail out.
2954 //
2955 // FIXME: We could try to reschedule the AND.
2956 return false;
2957 }
2958 } while (true);
2959
2960 // Return false if no candidates exist.
2961 if (!MI && !SubAdd)
2962 return false;
2963
2964 // If we found a SubAdd, use it as it will be closer to the CMP
2965 if (SubAdd) {
2966 MI = SubAdd;
2967 IsThumb1 = SubAddIsThumb1;
2968 }
2969
2970 // We can't use a predicated instruction - it doesn't always write the flags.
2971 if (isPredicated(MI: *MI))
2972 return false;
2973
2974 // Scan forward for the use of CPSR
2975 // When checking against MI: if it's a conditional code that requires
2976 // checking of the V bit or C bit, then this is not safe to do.
2977 // It is safe to remove CmpInstr if CPSR is redefined or killed.
2978 // If we are done with the basic block, we need to check whether CPSR is
2979 // live-out.
2980 SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
2981 OperandsToUpdate;
2982 bool isSafe = false;
2983 I = CmpInstr;
2984 E = CmpInstr.getParent()->end();
2985 while (!isSafe && ++I != E) {
2986 const MachineInstr &Instr = *I;
2987 for (unsigned IO = 0, EO = Instr.getNumOperands();
2988 !isSafe && IO != EO; ++IO) {
2989 const MachineOperand &MO = Instr.getOperand(i: IO);
2990 if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: ARM::CPSR)) {
2991 isSafe = true;
2992 break;
2993 }
2994 if (!MO.isReg() || MO.getReg() != ARM::CPSR)
2995 continue;
2996 if (MO.isDef()) {
2997 isSafe = true;
2998 break;
2999 }
3000 // Condition code is after the operand before CPSR except for VSELs.
3001 ARMCC::CondCodes CC;
3002 bool IsInstrVSel = true;
3003 switch (Instr.getOpcode()) {
3004 default:
3005 IsInstrVSel = false;
3006 CC = (ARMCC::CondCodes)Instr.getOperand(i: IO - 1).getImm();
3007 break;
3008 case ARM::VSELEQD:
3009 case ARM::VSELEQS:
3010 case ARM::VSELEQH:
3011 CC = ARMCC::EQ;
3012 break;
3013 case ARM::VSELGTD:
3014 case ARM::VSELGTS:
3015 case ARM::VSELGTH:
3016 CC = ARMCC::GT;
3017 break;
3018 case ARM::VSELGED:
3019 case ARM::VSELGES:
3020 case ARM::VSELGEH:
3021 CC = ARMCC::GE;
3022 break;
3023 case ARM::VSELVSD:
3024 case ARM::VSELVSS:
3025 case ARM::VSELVSH:
3026 CC = ARMCC::VS;
3027 break;
3028 }
3029
3030 if (SubAdd) {
3031 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3032 // on CMP needs to be updated to be based on SUB.
3033 // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3034 // needs to be modified.
3035 // Push the condition code operands to OperandsToUpdate.
3036 // If it is safe to remove CmpInstr, the condition code of these
3037 // operands will be modified.
3038 unsigned Opc = SubAdd->getOpcode();
3039 bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3040 Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3041 Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3042 Opc == ARM::tSUBi8;
3043 unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3044 if (!IsSub ||
3045 (SrcReg2 != 0 && SubAdd->getOperand(i: OpI).getReg() == SrcReg2 &&
3046 SubAdd->getOperand(i: OpI + 1).getReg() == SrcReg)) {
3047 // VSel doesn't support condition code update.
3048 if (IsInstrVSel)
3049 return false;
3050 // Ensure we can swap the condition.
3051 ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3052 if (NewCC == ARMCC::AL)
3053 return false;
3054 OperandsToUpdate.push_back(
3055 Elt: std::make_pair(x: &((*I).getOperand(i: IO - 1)), y&: NewCC));
3056 }
3057 } else {
3058 // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3059 switch (CC) {
3060 case ARMCC::EQ: // Z
3061 case ARMCC::NE: // Z
3062 case ARMCC::MI: // N
3063 case ARMCC::PL: // N
3064 case ARMCC::AL: // none
3065 // CPSR can be used multiple times, we should continue.
3066 break;
3067 case ARMCC::HS: // C
3068 case ARMCC::LO: // C
3069 case ARMCC::VS: // V
3070 case ARMCC::VC: // V
3071 case ARMCC::HI: // C Z
3072 case ARMCC::LS: // C Z
3073 case ARMCC::GE: // N V
3074 case ARMCC::LT: // N V
3075 case ARMCC::GT: // Z N V
3076 case ARMCC::LE: // Z N V
3077 // The instruction uses the V bit or C bit which is not safe.
3078 return false;
3079 }
3080 }
3081 }
3082 }
3083
3084 // If CPSR is not killed nor re-defined, we should check whether it is
3085 // live-out. If it is live-out, do not optimize.
3086 if (!isSafe) {
3087 MachineBasicBlock *MBB = CmpInstr.getParent();
3088 for (MachineBasicBlock *Succ : MBB->successors())
3089 if (Succ->isLiveIn(Reg: ARM::CPSR))
3090 return false;
3091 }
3092
3093 // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3094 // set CPSR so this is represented as an explicit output)
3095 if (!IsThumb1) {
3096 unsigned CPSRRegNum = MI->getNumExplicitOperands() - 1;
3097 MI->getOperand(i: CPSRRegNum).setReg(ARM::CPSR);
3098 MI->getOperand(i: CPSRRegNum).setIsDef(true);
3099 }
3100 assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3101 CmpInstr.eraseFromParent();
3102
3103 // Modify the condition code of operands in OperandsToUpdate.
3104 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3105 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3106 for (auto &[MO, Cond] : OperandsToUpdate)
3107 MO->setImm(Cond);
3108
3109 MI->clearRegisterDeads(Reg: ARM::CPSR);
3110
3111 return true;
3112}
3113
3114bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3115 // Do not sink MI if it might be used to optimize a redundant compare.
3116 // We heuristically only look at the instruction immediately following MI to
3117 // avoid potentially searching the entire basic block.
3118 if (isPredicated(MI))
3119 return true;
3120 MachineBasicBlock::const_iterator Next = &MI;
3121 ++Next;
3122 Register SrcReg, SrcReg2;
3123 int64_t CmpMask, CmpValue;
3124 bool IsThumb1;
3125 if (Next != MI.getParent()->end() &&
3126 analyzeCompare(MI: *Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3127 isRedundantFlagInstr(CmpI: &*Next, SrcReg, SrcReg2, ImmValue: CmpValue, OI: &MI, IsThumb1))
3128 return false;
3129 return true;
3130}
3131
3132bool ARMBaseInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3133 Register Reg,
3134 MachineRegisterInfo *MRI) const {
3135 // Fold large immediates into add, sub, or, xor.
3136 unsigned DefOpc = DefMI.getOpcode();
3137 if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm &&
3138 DefOpc != ARM::tMOVi32imm)
3139 return false;
3140 if (!DefMI.getOperand(i: 1).isImm())
3141 // Could be t2MOVi32imm @xx
3142 return false;
3143
3144 if (!MRI->hasOneNonDBGUse(RegNo: Reg))
3145 return false;
3146
3147 const MCInstrDesc &DefMCID = DefMI.getDesc();
3148 if (DefMCID.hasOptionalDef()) {
3149 unsigned NumOps = DefMCID.getNumOperands();
3150 const MachineOperand &MO = DefMI.getOperand(i: NumOps - 1);
3151 if (MO.getReg() == ARM::CPSR && !MO.isDead())
3152 // If DefMI defines CPSR and it is not dead, it's obviously not safe
3153 // to delete DefMI.
3154 return false;
3155 }
3156
3157 const MCInstrDesc &UseMCID = UseMI.getDesc();
3158 if (UseMCID.hasOptionalDef()) {
3159 unsigned NumOps = UseMCID.getNumOperands();
3160 if (UseMI.getOperand(i: NumOps - 1).getReg() == ARM::CPSR)
3161 // If the instruction sets the flag, do not attempt this optimization
3162 // since it may change the semantics of the code.
3163 return false;
3164 }
3165
3166 unsigned UseOpc = UseMI.getOpcode();
3167 unsigned NewUseOpc = 0;
3168 uint32_t ImmVal = (uint32_t)DefMI.getOperand(i: 1).getImm();
3169 uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3170 bool Commute = false;
3171 switch (UseOpc) {
3172 default: return false;
3173 case ARM::SUBrr:
3174 case ARM::ADDrr:
3175 case ARM::ORRrr:
3176 case ARM::EORrr:
3177 case ARM::t2SUBrr:
3178 case ARM::t2ADDrr:
3179 case ARM::t2ORRrr:
3180 case ARM::t2EORrr: {
3181 Commute = UseMI.getOperand(i: 2).getReg() != Reg;
3182 switch (UseOpc) {
3183 default: break;
3184 case ARM::ADDrr:
3185 case ARM::SUBrr:
3186 if (UseOpc == ARM::SUBrr && Commute)
3187 return false;
3188
3189 // ADD/SUB are special because they're essentially the same operation, so
3190 // we can handle a larger range of immediates.
3191 if (ARM_AM::isSOImmTwoPartVal(V: ImmVal))
3192 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3193 else if (ARM_AM::isSOImmTwoPartVal(V: -ImmVal)) {
3194 ImmVal = -ImmVal;
3195 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3196 } else
3197 return false;
3198 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(V: ImmVal);
3199 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(V: ImmVal);
3200 break;
3201 case ARM::ORRrr:
3202 case ARM::EORrr:
3203 if (!ARM_AM::isSOImmTwoPartVal(V: ImmVal))
3204 return false;
3205 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(V: ImmVal);
3206 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(V: ImmVal);
3207 switch (UseOpc) {
3208 default: break;
3209 case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3210 case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3211 }
3212 break;
3213 case ARM::t2ADDrr:
3214 case ARM::t2SUBrr: {
3215 if (UseOpc == ARM::t2SUBrr && Commute)
3216 return false;
3217
3218 // ADD/SUB are special because they're essentially the same operation, so
3219 // we can handle a larger range of immediates.
3220 const bool ToSP = DefMI.getOperand(i: 0).getReg() == ARM::SP;
3221 const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3222 const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3223 if (ARM_AM::isT2SOImmTwoPartVal(Imm: ImmVal))
3224 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3225 else if (ARM_AM::isT2SOImmTwoPartVal(Imm: -ImmVal)) {
3226 ImmVal = -ImmVal;
3227 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3228 } else
3229 return false;
3230 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(Imm: ImmVal);
3231 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(Imm: ImmVal);
3232 break;
3233 }
3234 case ARM::t2ORRrr:
3235 case ARM::t2EORrr:
3236 if (!ARM_AM::isT2SOImmTwoPartVal(Imm: ImmVal))
3237 return false;
3238 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(Imm: ImmVal);
3239 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(Imm: ImmVal);
3240 switch (UseOpc) {
3241 default: break;
3242 case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3243 case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3244 }
3245 break;
3246 }
3247 }
3248 }
3249
3250 unsigned OpIdx = Commute ? 2 : 1;
3251 Register Reg1 = UseMI.getOperand(i: OpIdx).getReg();
3252 bool isKill = UseMI.getOperand(i: OpIdx).isKill();
3253 const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3254 Register NewReg = MRI->createVirtualRegister(RegClass: TRC);
3255 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: NewUseOpc),
3256 DestReg: NewReg)
3257 .addReg(RegNo: Reg1, Flags: getKillRegState(B: isKill))
3258 .addImm(Val: SOImmValV1)
3259 .add(MOs: predOps(Pred: ARMCC::AL))
3260 .add(MO: condCodeOp());
3261 UseMI.setDesc(get(Opcode: NewUseOpc));
3262 UseMI.getOperand(i: 1).setReg(NewReg);
3263 UseMI.getOperand(i: 1).setIsKill();
3264 UseMI.getOperand(i: 2).ChangeToImmediate(ImmVal: SOImmValV2);
3265 DefMI.eraseFromParent();
3266 // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3267 // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3268 // Then the below code will not be needed, as the input/output register
3269 // classes will be rgpr or gprSP.
3270 // For now, we fix the UseMI operand explicitly here:
3271 switch(NewUseOpc){
3272 case ARM::t2ADDspImm:
3273 case ARM::t2SUBspImm:
3274 case ARM::t2ADDri:
3275 case ARM::t2SUBri:
3276 MRI->constrainRegClass(Reg: UseMI.getOperand(i: 0).getReg(), RC: TRC);
3277 }
3278 return true;
3279}
3280
3281static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3282 const MachineInstr &MI) {
3283 switch (MI.getOpcode()) {
3284 default: {
3285 const MCInstrDesc &Desc = MI.getDesc();
3286 int UOps = ItinData->getNumMicroOps(ItinClassIndx: Desc.getSchedClass());
3287 assert(UOps >= 0 && "bad # UOps");
3288 return UOps;
3289 }
3290
3291 case ARM::LDRrs:
3292 case ARM::LDRBrs:
3293 case ARM::STRrs:
3294 case ARM::STRBrs: {
3295 unsigned ShOpVal = MI.getOperand(i: 3).getImm();
3296 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3297 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3298 if (!isSub &&
3299 (ShImm == 0 ||
3300 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3301 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3302 return 1;
3303 return 2;
3304 }
3305
3306 case ARM::LDRH:
3307 case ARM::STRH: {
3308 if (!MI.getOperand(i: 2).getReg())
3309 return 1;
3310
3311 unsigned ShOpVal = MI.getOperand(i: 3).getImm();
3312 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3313 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3314 if (!isSub &&
3315 (ShImm == 0 ||
3316 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3317 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3318 return 1;
3319 return 2;
3320 }
3321
3322 case ARM::LDRSB:
3323 case ARM::LDRSH:
3324 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 3).getImm()) == ARM_AM::sub) ? 3 : 2;
3325
3326 case ARM::LDRSB_POST:
3327 case ARM::LDRSH_POST: {
3328 Register Rt = MI.getOperand(i: 0).getReg();
3329 Register Rm = MI.getOperand(i: 3).getReg();
3330 return (Rt == Rm) ? 4 : 3;
3331 }
3332
3333 case ARM::LDR_PRE_REG:
3334 case ARM::LDRB_PRE_REG: {
3335 Register Rt = MI.getOperand(i: 0).getReg();
3336 Register Rm = MI.getOperand(i: 3).getReg();
3337 if (Rt == Rm)
3338 return 3;
3339 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3340 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3341 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3342 if (!isSub &&
3343 (ShImm == 0 ||
3344 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3345 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3346 return 2;
3347 return 3;
3348 }
3349
3350 case ARM::STR_PRE_REG:
3351 case ARM::STRB_PRE_REG: {
3352 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3353 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3354 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3355 if (!isSub &&
3356 (ShImm == 0 ||
3357 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3358 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3359 return 2;
3360 return 3;
3361 }
3362
3363 case ARM::LDRH_PRE:
3364 case ARM::STRH_PRE: {
3365 Register Rt = MI.getOperand(i: 0).getReg();
3366 Register Rm = MI.getOperand(i: 3).getReg();
3367 if (!Rm)
3368 return 2;
3369 if (Rt == Rm)
3370 return 3;
3371 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 3 : 2;
3372 }
3373
3374 case ARM::LDR_POST_REG:
3375 case ARM::LDRB_POST_REG:
3376 case ARM::LDRH_POST: {
3377 Register Rt = MI.getOperand(i: 0).getReg();
3378 Register Rm = MI.getOperand(i: 3).getReg();
3379 return (Rt == Rm) ? 3 : 2;
3380 }
3381
3382 case ARM::LDR_PRE_IMM:
3383 case ARM::LDRB_PRE_IMM:
3384 case ARM::LDR_POST_IMM:
3385 case ARM::LDRB_POST_IMM:
3386 case ARM::STRB_POST_IMM:
3387 case ARM::STRB_POST_REG:
3388 case ARM::STRB_PRE_IMM:
3389 case ARM::STRH_POST:
3390 case ARM::STR_POST_IMM:
3391 case ARM::STR_POST_REG:
3392 case ARM::STR_PRE_IMM:
3393 return 2;
3394
3395 case ARM::LDRSB_PRE:
3396 case ARM::LDRSH_PRE: {
3397 Register Rm = MI.getOperand(i: 3).getReg();
3398 if (Rm == 0)
3399 return 3;
3400 Register Rt = MI.getOperand(i: 0).getReg();
3401 if (Rt == Rm)
3402 return 4;
3403 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3404 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3405 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3406 if (!isSub &&
3407 (ShImm == 0 ||
3408 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3409 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3410 return 3;
3411 return 4;
3412 }
3413
3414 case ARM::LDRD: {
3415 Register Rt = MI.getOperand(i: 0).getReg();
3416 Register Rn = MI.getOperand(i: 2).getReg();
3417 Register Rm = MI.getOperand(i: 3).getReg();
3418 if (Rm)
3419 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 4
3420 : 3;
3421 return (Rt == Rn) ? 3 : 2;
3422 }
3423
3424 case ARM::STRD: {
3425 Register Rm = MI.getOperand(i: 3).getReg();
3426 if (Rm)
3427 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 4
3428 : 3;
3429 return 2;
3430 }
3431
3432 case ARM::LDRD_POST:
3433 case ARM::t2LDRD_POST:
3434 return 3;
3435
3436 case ARM::STRD_POST:
3437 case ARM::t2STRD_POST:
3438 return 4;
3439
3440 case ARM::LDRD_PRE: {
3441 Register Rt = MI.getOperand(i: 0).getReg();
3442 Register Rn = MI.getOperand(i: 3).getReg();
3443 Register Rm = MI.getOperand(i: 4).getReg();
3444 if (Rm)
3445 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 5).getImm()) == ARM_AM::sub) ? 5
3446 : 4;
3447 return (Rt == Rn) ? 4 : 3;
3448 }
3449
3450 case ARM::t2LDRD_PRE: {
3451 Register Rt = MI.getOperand(i: 0).getReg();
3452 Register Rn = MI.getOperand(i: 3).getReg();
3453 return (Rt == Rn) ? 4 : 3;
3454 }
3455
3456 case ARM::STRD_PRE: {
3457 Register Rm = MI.getOperand(i: 4).getReg();
3458 if (Rm)
3459 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 5).getImm()) == ARM_AM::sub) ? 5
3460 : 4;
3461 return 3;
3462 }
3463
3464 case ARM::t2STRD_PRE:
3465 return 3;
3466
3467 case ARM::t2LDR_POST:
3468 case ARM::t2LDRB_POST:
3469 case ARM::t2LDRB_PRE:
3470 case ARM::t2LDRSBi12:
3471 case ARM::t2LDRSBi8:
3472 case ARM::t2LDRSBpci:
3473 case ARM::t2LDRSBs:
3474 case ARM::t2LDRH_POST:
3475 case ARM::t2LDRH_PRE:
3476 case ARM::t2LDRSBT:
3477 case ARM::t2LDRSB_POST:
3478 case ARM::t2LDRSB_PRE:
3479 case ARM::t2LDRSH_POST:
3480 case ARM::t2LDRSH_PRE:
3481 case ARM::t2LDRSHi12:
3482 case ARM::t2LDRSHi8:
3483 case ARM::t2LDRSHpci:
3484 case ARM::t2LDRSHs:
3485 return 2;
3486
3487 case ARM::t2LDRDi8: {
3488 Register Rt = MI.getOperand(i: 0).getReg();
3489 Register Rn = MI.getOperand(i: 2).getReg();
3490 return (Rt == Rn) ? 3 : 2;
3491 }
3492
3493 case ARM::t2STRB_POST:
3494 case ARM::t2STRB_PRE:
3495 case ARM::t2STRBs:
3496 case ARM::t2STRDi8:
3497 case ARM::t2STRH_POST:
3498 case ARM::t2STRH_PRE:
3499 case ARM::t2STRHs:
3500 case ARM::t2STR_POST:
3501 case ARM::t2STR_PRE:
3502 case ARM::t2STRs:
3503 return 2;
3504 }
3505}
3506
3507// Return the number of 32-bit words loaded by LDM or stored by STM. If this
3508// can't be easily determined return 0 (missing MachineMemOperand).
3509//
3510// FIXME: The current MachineInstr design does not support relying on machine
3511// mem operands to determine the width of a memory access. Instead, we expect
3512// the target to provide this information based on the instruction opcode and
3513// operands. However, using MachineMemOperand is the best solution now for
3514// two reasons:
3515//
3516// 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3517// operands. This is much more dangerous than using the MachineMemOperand
3518// sizes because CodeGen passes can insert/remove optional machine operands. In
3519// fact, it's totally incorrect for preRA passes and appears to be wrong for
3520// postRA passes as well.
3521//
3522// 2) getNumLDMAddresses is only used by the scheduling machine model and any
3523// machine model that calls this should handle the unknown (zero size) case.
3524//
3525// Long term, we should require a target hook that verifies MachineMemOperand
3526// sizes during MC lowering. That target hook should be local to MC lowering
3527// because we can't ensure that it is aware of other MI forms. Doing this will
3528// ensure that MachineMemOperands are correctly propagated through all passes.
3529unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3530 unsigned Size = 0;
3531 for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3532 E = MI.memoperands_end();
3533 I != E; ++I) {
3534 Size += (*I)->getSize().getValue();
3535 }
3536 // FIXME: The scheduler currently can't handle values larger than 16. But
3537 // the values can actually go up to 32 for floating-point load/store
3538 // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3539 // operations isn't right; we could end up with "extra" memory operands for
3540 // various reasons, like tail merge merging two memory operations.
3541 return std::min(a: Size / 4, b: 16U);
3542}
3543
3544static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3545 unsigned NumRegs) {
3546 unsigned UOps = 1 + NumRegs; // 1 for address computation.
3547 switch (Opc) {
3548 default:
3549 break;
3550 case ARM::VLDMDIA_UPD:
3551 case ARM::VLDMDDB_UPD:
3552 case ARM::VLDMSIA_UPD:
3553 case ARM::VLDMSDB_UPD:
3554 case ARM::VSTMDIA_UPD:
3555 case ARM::VSTMDDB_UPD:
3556 case ARM::VSTMSIA_UPD:
3557 case ARM::VSTMSDB_UPD:
3558 case ARM::LDMIA_UPD:
3559 case ARM::LDMDA_UPD:
3560 case ARM::LDMDB_UPD:
3561 case ARM::LDMIB_UPD:
3562 case ARM::STMIA_UPD:
3563 case ARM::STMDA_UPD:
3564 case ARM::STMDB_UPD:
3565 case ARM::STMIB_UPD:
3566 case ARM::tLDMIA_UPD:
3567 case ARM::tSTMIA_UPD:
3568 case ARM::t2LDMIA_UPD:
3569 case ARM::t2LDMDB_UPD:
3570 case ARM::t2STMIA_UPD:
3571 case ARM::t2STMDB_UPD:
3572 ++UOps; // One for base register writeback.
3573 break;
3574 case ARM::LDMIA_RET:
3575 case ARM::tPOP_RET:
3576 case ARM::t2LDMIA_RET:
3577 UOps += 2; // One for base reg wb, one for write to pc.
3578 break;
3579 }
3580 return UOps;
3581}
3582
3583unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3584 const MachineInstr &MI) const {
3585 if (!ItinData || ItinData->isEmpty())
3586 return 1;
3587
3588 const MCInstrDesc &Desc = MI.getDesc();
3589 unsigned Class = Desc.getSchedClass();
3590 int ItinUOps = ItinData->getNumMicroOps(ItinClassIndx: Class);
3591 if (ItinUOps >= 0) {
3592 if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3593 return getNumMicroOpsSwiftLdSt(ItinData, MI);
3594
3595 return ItinUOps;
3596 }
3597
3598 unsigned Opc = MI.getOpcode();
3599 switch (Opc) {
3600 default:
3601 llvm_unreachable("Unexpected multi-uops instruction!");
3602 case ARM::VLDMQIA:
3603 case ARM::VSTMQIA:
3604 return 2;
3605
3606 // The number of uOps for load / store multiple are determined by the number
3607 // registers.
3608 //
3609 // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3610 // same cycle. The scheduling for the first load / store must be done
3611 // separately by assuming the address is not 64-bit aligned.
3612 //
3613 // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3614 // is not 64-bit aligned, then AGU would take an extra cycle. For VFP / NEON
3615 // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3616 case ARM::VLDMDIA:
3617 case ARM::VLDMDIA_UPD:
3618 case ARM::VLDMDDB_UPD:
3619 case ARM::VLDMSIA:
3620 case ARM::VLDMSIA_UPD:
3621 case ARM::VLDMSDB_UPD:
3622 case ARM::VSTMDIA:
3623 case ARM::VSTMDIA_UPD:
3624 case ARM::VSTMDDB_UPD:
3625 case ARM::VSTMSIA:
3626 case ARM::VSTMSIA_UPD:
3627 case ARM::VSTMSDB_UPD: {
3628 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3629 return (NumRegs / 2) + (NumRegs % 2) + 1;
3630 }
3631
3632 case ARM::LDMIA_RET:
3633 case ARM::LDMIA:
3634 case ARM::LDMDA:
3635 case ARM::LDMDB:
3636 case ARM::LDMIB:
3637 case ARM::LDMIA_UPD:
3638 case ARM::LDMDA_UPD:
3639 case ARM::LDMDB_UPD:
3640 case ARM::LDMIB_UPD:
3641 case ARM::STMIA:
3642 case ARM::STMDA:
3643 case ARM::STMDB:
3644 case ARM::STMIB:
3645 case ARM::STMIA_UPD:
3646 case ARM::STMDA_UPD:
3647 case ARM::STMDB_UPD:
3648 case ARM::STMIB_UPD:
3649 case ARM::tLDMIA:
3650 case ARM::tLDMIA_UPD:
3651 case ARM::tSTMIA_UPD:
3652 case ARM::tPOP_RET:
3653 case ARM::tPOP:
3654 case ARM::tPUSH:
3655 case ARM::t2LDMIA_RET:
3656 case ARM::t2LDMIA:
3657 case ARM::t2LDMDB:
3658 case ARM::t2LDMIA_UPD:
3659 case ARM::t2LDMDB_UPD:
3660 case ARM::t2STMIA:
3661 case ARM::t2STMDB:
3662 case ARM::t2STMIA_UPD:
3663 case ARM::t2STMDB_UPD: {
3664 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3665 switch (Subtarget.getLdStMultipleTiming()) {
3666 case ARMSubtarget::SingleIssuePlusExtras:
3667 return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3668 case ARMSubtarget::SingleIssue:
3669 // Assume the worst.
3670 return NumRegs;
3671 case ARMSubtarget::DoubleIssue: {
3672 if (NumRegs < 4)
3673 return 2;
3674 // 4 registers would be issued: 2, 2.
3675 // 5 registers would be issued: 2, 2, 1.
3676 unsigned UOps = (NumRegs / 2);
3677 if (NumRegs % 2)
3678 ++UOps;
3679 return UOps;
3680 }
3681 case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3682 unsigned UOps = (NumRegs / 2);
3683 // If there are odd number of registers or if it's not 64-bit aligned,
3684 // then it takes an extra AGU (Address Generation Unit) cycle.
3685 if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3686 (*MI.memoperands_begin())->getAlign() < Align(8))
3687 ++UOps;
3688 return UOps;
3689 }
3690 }
3691 }
3692 }
3693 llvm_unreachable("Didn't find the number of microops");
3694}
3695
3696std::optional<unsigned>
3697ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3698 const MCInstrDesc &DefMCID, unsigned DefClass,
3699 unsigned DefIdx, unsigned DefAlign) const {
3700 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3701 if (RegNo <= 0)
3702 // Def is the address writeback.
3703 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3704
3705 unsigned DefCycle;
3706 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3707 // (regno / 2) + (regno % 2) + 1
3708 DefCycle = RegNo / 2 + 1;
3709 if (RegNo % 2)
3710 ++DefCycle;
3711 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3712 DefCycle = RegNo;
3713 bool isSLoad = false;
3714
3715 switch (DefMCID.getOpcode()) {
3716 default: break;
3717 case ARM::VLDMSIA:
3718 case ARM::VLDMSIA_UPD:
3719 case ARM::VLDMSDB_UPD:
3720 isSLoad = true;
3721 break;
3722 }
3723
3724 // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3725 // then it takes an extra cycle.
3726 if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3727 ++DefCycle;
3728 } else {
3729 // Assume the worst.
3730 DefCycle = RegNo + 2;
3731 }
3732
3733 return DefCycle;
3734}
3735
3736std::optional<unsigned>
3737ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3738 const MCInstrDesc &DefMCID, unsigned DefClass,
3739 unsigned DefIdx, unsigned DefAlign) const {
3740 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3741 if (RegNo <= 0)
3742 // Def is the address writeback.
3743 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3744
3745 unsigned DefCycle;
3746 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3747 // 4 registers would be issued: 1, 2, 1.
3748 // 5 registers would be issued: 1, 2, 2.
3749 DefCycle = RegNo / 2;
3750 if (DefCycle < 1)
3751 DefCycle = 1;
3752 // Result latency is issue cycle + 2: E2.
3753 DefCycle += 2;
3754 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3755 DefCycle = (RegNo / 2);
3756 // If there are odd number of registers or if it's not 64-bit aligned,
3757 // then it takes an extra AGU (Address Generation Unit) cycle.
3758 if ((RegNo % 2) || DefAlign < 8)
3759 ++DefCycle;
3760 // Result latency is AGU cycles + 2.
3761 DefCycle += 2;
3762 } else {
3763 // Assume the worst.
3764 DefCycle = RegNo + 2;
3765 }
3766
3767 return DefCycle;
3768}
3769
3770std::optional<unsigned>
3771ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3772 const MCInstrDesc &UseMCID, unsigned UseClass,
3773 unsigned UseIdx, unsigned UseAlign) const {
3774 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3775 if (RegNo <= 0)
3776 return ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3777
3778 unsigned UseCycle;
3779 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3780 // (regno / 2) + (regno % 2) + 1
3781 UseCycle = RegNo / 2 + 1;
3782 if (RegNo % 2)
3783 ++UseCycle;
3784 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3785 UseCycle = RegNo;
3786 bool isSStore = false;
3787
3788 switch (UseMCID.getOpcode()) {
3789 default: break;
3790 case ARM::VSTMSIA:
3791 case ARM::VSTMSIA_UPD:
3792 case ARM::VSTMSDB_UPD:
3793 isSStore = true;
3794 break;
3795 }
3796
3797 // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3798 // then it takes an extra cycle.
3799 if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3800 ++UseCycle;
3801 } else {
3802 // Assume the worst.
3803 UseCycle = RegNo + 2;
3804 }
3805
3806 return UseCycle;
3807}
3808
3809std::optional<unsigned>
3810ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3811 const MCInstrDesc &UseMCID, unsigned UseClass,
3812 unsigned UseIdx, unsigned UseAlign) const {
3813 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3814 if (RegNo <= 0)
3815 return ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3816
3817 unsigned UseCycle;
3818 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3819 UseCycle = RegNo / 2;
3820 if (UseCycle < 2)
3821 UseCycle = 2;
3822 // Read in E3.
3823 UseCycle += 2;
3824 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3825 UseCycle = (RegNo / 2);
3826 // If there are odd number of registers or if it's not 64-bit aligned,
3827 // then it takes an extra AGU (Address Generation Unit) cycle.
3828 if ((RegNo % 2) || UseAlign < 8)
3829 ++UseCycle;
3830 } else {
3831 // Assume the worst.
3832 UseCycle = 1;
3833 }
3834 return UseCycle;
3835}
3836
3837std::optional<unsigned> ARMBaseInstrInfo::getOperandLatency(
3838 const InstrItineraryData *ItinData, const MCInstrDesc &DefMCID,
3839 unsigned DefIdx, unsigned DefAlign, const MCInstrDesc &UseMCID,
3840 unsigned UseIdx, unsigned UseAlign) const {
3841 unsigned DefClass = DefMCID.getSchedClass();
3842 unsigned UseClass = UseMCID.getSchedClass();
3843
3844 if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3845 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3846
3847 // This may be a def / use of a variable_ops instruction, the operand
3848 // latency might be determinable dynamically. Let the target try to
3849 // figure it out.
3850 std::optional<unsigned> DefCycle;
3851 bool LdmBypass = false;
3852 switch (DefMCID.getOpcode()) {
3853 default:
3854 DefCycle = ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3855 break;
3856
3857 case ARM::VLDMDIA:
3858 case ARM::VLDMDIA_UPD:
3859 case ARM::VLDMDDB_UPD:
3860 case ARM::VLDMSIA:
3861 case ARM::VLDMSIA_UPD:
3862 case ARM::VLDMSDB_UPD:
3863 DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3864 break;
3865
3866 case ARM::LDMIA_RET:
3867 case ARM::LDMIA:
3868 case ARM::LDMDA:
3869 case ARM::LDMDB:
3870 case ARM::LDMIB:
3871 case ARM::LDMIA_UPD:
3872 case ARM::LDMDA_UPD:
3873 case ARM::LDMDB_UPD:
3874 case ARM::LDMIB_UPD:
3875 case ARM::tLDMIA:
3876 case ARM::tLDMIA_UPD:
3877 case ARM::tPUSH:
3878 case ARM::t2LDMIA_RET:
3879 case ARM::t2LDMIA:
3880 case ARM::t2LDMDB:
3881 case ARM::t2LDMIA_UPD:
3882 case ARM::t2LDMDB_UPD:
3883 LdmBypass = true;
3884 DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3885 break;
3886 }
3887
3888 if (!DefCycle)
3889 // We can't seem to determine the result latency of the def, assume it's 2.
3890 DefCycle = 2;
3891
3892 std::optional<unsigned> UseCycle;
3893 switch (UseMCID.getOpcode()) {
3894 default:
3895 UseCycle = ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3896 break;
3897
3898 case ARM::VSTMDIA:
3899 case ARM::VSTMDIA_UPD:
3900 case ARM::VSTMDDB_UPD:
3901 case ARM::VSTMSIA:
3902 case ARM::VSTMSIA_UPD:
3903 case ARM::VSTMSDB_UPD:
3904 UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3905 break;
3906
3907 case ARM::STMIA:
3908 case ARM::STMDA:
3909 case ARM::STMDB:
3910 case ARM::STMIB:
3911 case ARM::STMIA_UPD:
3912 case ARM::STMDA_UPD:
3913 case ARM::STMDB_UPD:
3914 case ARM::STMIB_UPD:
3915 case ARM::tSTMIA_UPD:
3916 case ARM::tPOP_RET:
3917 case ARM::tPOP:
3918 case ARM::t2STMIA:
3919 case ARM::t2STMDB:
3920 case ARM::t2STMIA_UPD:
3921 case ARM::t2STMDB_UPD:
3922 UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3923 break;
3924 }
3925
3926 if (!UseCycle)
3927 // Assume it's read in the first stage.
3928 UseCycle = 1;
3929
3930 if (UseCycle > *DefCycle + 1)
3931 return std::nullopt;
3932
3933 UseCycle = *DefCycle - *UseCycle + 1;
3934 if (UseCycle > 0u) {
3935 if (LdmBypass) {
3936 // It's a variable_ops instruction so we can't use DefIdx here. Just use
3937 // first def operand.
3938 if (ItinData->hasPipelineForwarding(DefClass, DefIdx: DefMCID.getNumOperands()-1,
3939 UseClass, UseIdx))
3940 UseCycle = *UseCycle - 1;
3941 } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
3942 UseClass, UseIdx)) {
3943 UseCycle = *UseCycle - 1;
3944 }
3945 }
3946
3947 return UseCycle;
3948}
3949
3950static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
3951 const MachineInstr *MI, unsigned Reg,
3952 unsigned &DefIdx, unsigned &Dist) {
3953 Dist = 0;
3954
3955 MachineBasicBlock::const_iterator I = MI; ++I;
3956 MachineBasicBlock::const_instr_iterator II = std::prev(x: I.getInstrIterator());
3957 assert(II->isInsideBundle() && "Empty bundle?");
3958
3959 int Idx = -1;
3960 while (II->isInsideBundle()) {
3961 Idx = II->findRegisterDefOperandIdx(Reg, TRI, isDead: false, Overlap: true);
3962 if (Idx != -1)
3963 break;
3964 --II;
3965 ++Dist;
3966 }
3967
3968 assert(Idx != -1 && "Cannot find bundled definition!");
3969 DefIdx = Idx;
3970 return &*II;
3971}
3972
3973static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
3974 const MachineInstr &MI, unsigned Reg,
3975 unsigned &UseIdx, unsigned &Dist) {
3976 Dist = 0;
3977
3978 MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
3979 assert(II->isInsideBundle() && "Empty bundle?");
3980 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
3981
3982 // FIXME: This doesn't properly handle multiple uses.
3983 int Idx = -1;
3984 while (II != E && II->isInsideBundle()) {
3985 Idx = II->findRegisterUseOperandIdx(Reg, TRI, isKill: false);
3986 if (Idx != -1)
3987 break;
3988 if (II->getOpcode() != ARM::t2IT)
3989 ++Dist;
3990 ++II;
3991 }
3992
3993 if (Idx == -1) {
3994 Dist = 0;
3995 return nullptr;
3996 }
3997
3998 UseIdx = Idx;
3999 return &*II;
4000}
4001
4002/// Return the number of cycles to add to (or subtract from) the static
4003/// itinerary based on the def opcode and alignment. The caller will ensure that
4004/// adjusted latency is at least one cycle.
4005static int adjustDefLatency(const ARMSubtarget &Subtarget,
4006 const MachineInstr &DefMI,
4007 const MCInstrDesc &DefMCID, unsigned DefAlign) {
4008 int Adjust = 0;
4009 if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4010 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4011 // variants are one cycle cheaper.
4012 switch (DefMCID.getOpcode()) {
4013 default: break;
4014 case ARM::LDRrs:
4015 case ARM::LDRBrs: {
4016 unsigned ShOpVal = DefMI.getOperand(i: 3).getImm();
4017 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4018 if (ShImm == 0 ||
4019 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4020 --Adjust;
4021 break;
4022 }
4023 case ARM::t2LDRs:
4024 case ARM::t2LDRBs:
4025 case ARM::t2LDRHs:
4026 case ARM::t2LDRSHs: {
4027 // Thumb2 mode: lsl only.
4028 unsigned ShAmt = DefMI.getOperand(i: 3).getImm();
4029 if (ShAmt == 0 || ShAmt == 2)
4030 --Adjust;
4031 break;
4032 }
4033 }
4034 } else if (Subtarget.isSwift()) {
4035 // FIXME: Properly handle all of the latency adjustments for address
4036 // writeback.
4037 switch (DefMCID.getOpcode()) {
4038 default: break;
4039 case ARM::LDRrs:
4040 case ARM::LDRBrs: {
4041 unsigned ShOpVal = DefMI.getOperand(i: 3).getImm();
4042 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
4043 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4044 if (!isSub &&
4045 (ShImm == 0 ||
4046 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4047 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
4048 Adjust -= 2;
4049 else if (!isSub &&
4050 ShImm == 1 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsr)
4051 --Adjust;
4052 break;
4053 }
4054 case ARM::t2LDRs:
4055 case ARM::t2LDRBs:
4056 case ARM::t2LDRHs:
4057 case ARM::t2LDRSHs: {
4058 // Thumb2 mode: lsl only.
4059 unsigned ShAmt = DefMI.getOperand(i: 3).getImm();
4060 if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4061 Adjust -= 2;
4062 break;
4063 }
4064 }
4065 }
4066
4067 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4068 switch (DefMCID.getOpcode()) {
4069 default: break;
4070 case ARM::VLD1q8:
4071 case ARM::VLD1q16:
4072 case ARM::VLD1q32:
4073 case ARM::VLD1q64:
4074 case ARM::VLD1q8wb_fixed:
4075 case ARM::VLD1q16wb_fixed:
4076 case ARM::VLD1q32wb_fixed:
4077 case ARM::VLD1q64wb_fixed:
4078 case ARM::VLD1q8wb_register:
4079 case ARM::VLD1q16wb_register:
4080 case ARM::VLD1q32wb_register:
4081 case ARM::VLD1q64wb_register:
4082 case ARM::VLD2d8:
4083 case ARM::VLD2d16:
4084 case ARM::VLD2d32:
4085 case ARM::VLD2q8:
4086 case ARM::VLD2q16:
4087 case ARM::VLD2q32:
4088 case ARM::VLD2d8wb_fixed:
4089 case ARM::VLD2d16wb_fixed:
4090 case ARM::VLD2d32wb_fixed:
4091 case ARM::VLD2q8wb_fixed:
4092 case ARM::VLD2q16wb_fixed:
4093 case ARM::VLD2q32wb_fixed:
4094 case ARM::VLD2d8wb_register:
4095 case ARM::VLD2d16wb_register:
4096 case ARM::VLD2d32wb_register:
4097 case ARM::VLD2q8wb_register:
4098 case ARM::VLD2q16wb_register:
4099 case ARM::VLD2q32wb_register:
4100 case ARM::VLD3d8:
4101 case ARM::VLD3d16:
4102 case ARM::VLD3d32:
4103 case ARM::VLD1d64T:
4104 case ARM::VLD3d8_UPD:
4105 case ARM::VLD3d16_UPD:
4106 case ARM::VLD3d32_UPD:
4107 case ARM::VLD1d64Twb_fixed:
4108 case ARM::VLD1d64Twb_register:
4109 case ARM::VLD3q8_UPD:
4110 case ARM::VLD3q16_UPD:
4111 case ARM::VLD3q32_UPD:
4112 case ARM::VLD4d8:
4113 case ARM::VLD4d16:
4114 case ARM::VLD4d32:
4115 case ARM::VLD1d64Q:
4116 case ARM::VLD4d8_UPD:
4117 case ARM::VLD4d16_UPD:
4118 case ARM::VLD4d32_UPD:
4119 case ARM::VLD1d64Qwb_fixed:
4120 case ARM::VLD1d64Qwb_register:
4121 case ARM::VLD4q8_UPD:
4122 case ARM::VLD4q16_UPD:
4123 case ARM::VLD4q32_UPD:
4124 case ARM::VLD1DUPq8:
4125 case ARM::VLD1DUPq16:
4126 case ARM::VLD1DUPq32:
4127 case ARM::VLD1DUPq8wb_fixed:
4128 case ARM::VLD1DUPq16wb_fixed:
4129 case ARM::VLD1DUPq32wb_fixed:
4130 case ARM::VLD1DUPq8wb_register:
4131 case ARM::VLD1DUPq16wb_register:
4132 case ARM::VLD1DUPq32wb_register:
4133 case ARM::VLD2DUPd8:
4134 case ARM::VLD2DUPd16:
4135 case ARM::VLD2DUPd32:
4136 case ARM::VLD2DUPd8wb_fixed:
4137 case ARM::VLD2DUPd16wb_fixed:
4138 case ARM::VLD2DUPd32wb_fixed:
4139 case ARM::VLD2DUPd8wb_register:
4140 case ARM::VLD2DUPd16wb_register:
4141 case ARM::VLD2DUPd32wb_register:
4142 case ARM::VLD4DUPd8:
4143 case ARM::VLD4DUPd16:
4144 case ARM::VLD4DUPd32:
4145 case ARM::VLD4DUPd8_UPD:
4146 case ARM::VLD4DUPd16_UPD:
4147 case ARM::VLD4DUPd32_UPD:
4148 case ARM::VLD1LNd8:
4149 case ARM::VLD1LNd16:
4150 case ARM::VLD1LNd32:
4151 case ARM::VLD1LNd8_UPD:
4152 case ARM::VLD1LNd16_UPD:
4153 case ARM::VLD1LNd32_UPD:
4154 case ARM::VLD2LNd8:
4155 case ARM::VLD2LNd16:
4156 case ARM::VLD2LNd32:
4157 case ARM::VLD2LNq16:
4158 case ARM::VLD2LNq32:
4159 case ARM::VLD2LNd8_UPD:
4160 case ARM::VLD2LNd16_UPD:
4161 case ARM::VLD2LNd32_UPD:
4162 case ARM::VLD2LNq16_UPD:
4163 case ARM::VLD2LNq32_UPD:
4164 case ARM::VLD4LNd8:
4165 case ARM::VLD4LNd16:
4166 case ARM::VLD4LNd32:
4167 case ARM::VLD4LNq16:
4168 case ARM::VLD4LNq32:
4169 case ARM::VLD4LNd8_UPD:
4170 case ARM::VLD4LNd16_UPD:
4171 case ARM::VLD4LNd32_UPD:
4172 case ARM::VLD4LNq16_UPD:
4173 case ARM::VLD4LNq32_UPD:
4174 // If the address is not 64-bit aligned, the latencies of these
4175 // instructions increases by one.
4176 ++Adjust;
4177 break;
4178 }
4179 }
4180 return Adjust;
4181}
4182
4183std::optional<unsigned> ARMBaseInstrInfo::getOperandLatency(
4184 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4185 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
4186 // No operand latency. The caller may fall back to getInstrLatency.
4187 if (!ItinData || ItinData->isEmpty())
4188 return std::nullopt;
4189
4190 const MachineOperand &DefMO = DefMI.getOperand(i: DefIdx);
4191 Register Reg = DefMO.getReg();
4192
4193 const MachineInstr *ResolvedDefMI = &DefMI;
4194 unsigned DefAdj = 0;
4195 if (DefMI.isBundle())
4196 ResolvedDefMI =
4197 getBundledDefMI(TRI: &getRegisterInfo(), MI: &DefMI, Reg, DefIdx, Dist&: DefAdj);
4198 if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4199 ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4200 return 1;
4201 }
4202
4203 const MachineInstr *ResolvedUseMI = &UseMI;
4204 unsigned UseAdj = 0;
4205 if (UseMI.isBundle()) {
4206 ResolvedUseMI =
4207 getBundledUseMI(TRI: &getRegisterInfo(), MI: UseMI, Reg, UseIdx, Dist&: UseAdj);
4208 if (!ResolvedUseMI)
4209 return std::nullopt;
4210 }
4211
4212 return getOperandLatencyImpl(
4213 ItinData, DefMI: *ResolvedDefMI, DefIdx, DefMCID: ResolvedDefMI->getDesc(), DefAdj, DefMO,
4214 Reg, UseMI: *ResolvedUseMI, UseIdx, UseMCID: ResolvedUseMI->getDesc(), UseAdj);
4215}
4216
4217std::optional<unsigned> ARMBaseInstrInfo::getOperandLatencyImpl(
4218 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4219 unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4220 const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4221 unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4222 if (Reg == ARM::CPSR) {
4223 if (DefMI.getOpcode() == ARM::FMSTAT) {
4224 // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4225 return Subtarget.isLikeA9() ? 1 : 20;
4226 }
4227
4228 // CPSR set and branch can be paired in the same cycle.
4229 if (UseMI.isBranch())
4230 return 0;
4231
4232 // Otherwise it takes the instruction latency (generally one).
4233 unsigned Latency = getInstrLatency(ItinData, MI: DefMI);
4234
4235 // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4236 // its uses. Instructions which are otherwise scheduled between them may
4237 // incur a code size penalty (not able to use the CPSR setting 16-bit
4238 // instructions).
4239 if (Latency > 0 && Subtarget.isThumb2()) {
4240 const MachineFunction *MF = DefMI.getParent()->getParent();
4241 if (MF->getFunction().hasOptSize())
4242 --Latency;
4243 }
4244 return Latency;
4245 }
4246
4247 if (DefMO.isImplicit() || UseMI.getOperand(i: UseIdx).isImplicit())
4248 return std::nullopt;
4249
4250 unsigned DefAlign = DefMI.hasOneMemOperand()
4251 ? (*DefMI.memoperands_begin())->getAlign().value()
4252 : 0;
4253 unsigned UseAlign = UseMI.hasOneMemOperand()
4254 ? (*UseMI.memoperands_begin())->getAlign().value()
4255 : 0;
4256
4257 // Get the itinerary's latency if possible, and handle variable_ops.
4258 std::optional<unsigned> Latency = getOperandLatency(
4259 ItinData, DefMCID, DefIdx, DefAlign, UseMCID, UseIdx, UseAlign);
4260 // Unable to find operand latency. The caller may resort to getInstrLatency.
4261 if (!Latency)
4262 return std::nullopt;
4263
4264 // Adjust for IT block position.
4265 int Adj = DefAdj + UseAdj;
4266
4267 // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4268 Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4269 if (Adj >= 0 || (int)*Latency > -Adj) {
4270 return *Latency + Adj;
4271 }
4272 // Return the itinerary latency, which may be zero but not less than zero.
4273 return Latency;
4274}
4275
4276std::optional<unsigned>
4277ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4278 SDNode *DefNode, unsigned DefIdx,
4279 SDNode *UseNode, unsigned UseIdx) const {
4280 if (!DefNode->isMachineOpcode())
4281 return 1;
4282
4283 const MCInstrDesc &DefMCID = get(Opcode: DefNode->getMachineOpcode());
4284
4285 if (isZeroCost(Opcode: DefMCID.Opcode))
4286 return 0;
4287
4288 if (!ItinData || ItinData->isEmpty())
4289 return DefMCID.mayLoad() ? 3 : 1;
4290
4291 if (!UseNode->isMachineOpcode()) {
4292 std::optional<unsigned> Latency =
4293 ItinData->getOperandCycle(ItinClassIndx: DefMCID.getSchedClass(), OperandIdx: DefIdx);
4294 int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4295 int Threshold = 1 + Adj;
4296 return !Latency || Latency <= (unsigned)Threshold ? 1 : *Latency - Adj;
4297 }
4298
4299 const MCInstrDesc &UseMCID = get(Opcode: UseNode->getMachineOpcode());
4300 auto *DefMN = cast<MachineSDNode>(Val: DefNode);
4301 unsigned DefAlign = !DefMN->memoperands_empty()
4302 ? (*DefMN->memoperands_begin())->getAlign().value()
4303 : 0;
4304 auto *UseMN = cast<MachineSDNode>(Val: UseNode);
4305 unsigned UseAlign = !UseMN->memoperands_empty()
4306 ? (*UseMN->memoperands_begin())->getAlign().value()
4307 : 0;
4308 std::optional<unsigned> Latency = getOperandLatency(
4309 ItinData, DefMCID, DefIdx, DefAlign, UseMCID, UseIdx, UseAlign);
4310 if (!Latency)
4311 return std::nullopt;
4312
4313 if (Latency > 1U &&
4314 (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4315 Subtarget.isCortexA7())) {
4316 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4317 // variants are one cycle cheaper.
4318 switch (DefMCID.getOpcode()) {
4319 default: break;
4320 case ARM::LDRrs:
4321 case ARM::LDRBrs: {
4322 unsigned ShOpVal = DefNode->getConstantOperandVal(Num: 2);
4323 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4324 if (ShImm == 0 ||
4325 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4326 Latency = *Latency - 1;
4327 break;
4328 }
4329 case ARM::t2LDRs:
4330 case ARM::t2LDRBs:
4331 case ARM::t2LDRHs:
4332 case ARM::t2LDRSHs: {
4333 // Thumb2 mode: lsl only.
4334 unsigned ShAmt = DefNode->getConstantOperandVal(Num: 2);
4335 if (ShAmt == 0 || ShAmt == 2)
4336 Latency = *Latency - 1;
4337 break;
4338 }
4339 }
4340 } else if (DefIdx == 0 && Latency > 2U && Subtarget.isSwift()) {
4341 // FIXME: Properly handle all of the latency adjustments for address
4342 // writeback.
4343 switch (DefMCID.getOpcode()) {
4344 default: break;
4345 case ARM::LDRrs:
4346 case ARM::LDRBrs: {
4347 unsigned ShOpVal = DefNode->getConstantOperandVal(Num: 2);
4348 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4349 if (ShImm == 0 ||
4350 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4351 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4352 Latency = *Latency - 2;
4353 else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsr)
4354 Latency = *Latency - 1;
4355 break;
4356 }
4357 case ARM::t2LDRs:
4358 case ARM::t2LDRBs:
4359 case ARM::t2LDRHs:
4360 case ARM::t2LDRSHs:
4361 // Thumb2 mode: lsl 0-3 only.
4362 Latency = *Latency - 2;
4363 break;
4364 }
4365 }
4366
4367 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4368 switch (DefMCID.getOpcode()) {
4369 default: break;
4370 case ARM::VLD1q8:
4371 case ARM::VLD1q16:
4372 case ARM::VLD1q32:
4373 case ARM::VLD1q64:
4374 case ARM::VLD1q8wb_register:
4375 case ARM::VLD1q16wb_register:
4376 case ARM::VLD1q32wb_register:
4377 case ARM::VLD1q64wb_register:
4378 case ARM::VLD1q8wb_fixed:
4379 case ARM::VLD1q16wb_fixed:
4380 case ARM::VLD1q32wb_fixed:
4381 case ARM::VLD1q64wb_fixed:
4382 case ARM::VLD2d8:
4383 case ARM::VLD2d16:
4384 case ARM::VLD2d32:
4385 case ARM::VLD2q8Pseudo:
4386 case ARM::VLD2q16Pseudo:
4387 case ARM::VLD2q32Pseudo:
4388 case ARM::VLD2d8wb_fixed:
4389 case ARM::VLD2d16wb_fixed:
4390 case ARM::VLD2d32wb_fixed:
4391 case ARM::VLD2q8PseudoWB_fixed:
4392 case ARM::VLD2q16PseudoWB_fixed:
4393 case ARM::VLD2q32PseudoWB_fixed:
4394 case ARM::VLD2d8wb_register:
4395 case ARM::VLD2d16wb_register:
4396 case ARM::VLD2d32wb_register:
4397 case ARM::VLD2q8PseudoWB_register:
4398 case ARM::VLD2q16PseudoWB_register:
4399 case ARM::VLD2q32PseudoWB_register:
4400 case ARM::VLD3d8Pseudo:
4401 case ARM::VLD3d16Pseudo:
4402 case ARM::VLD3d32Pseudo:
4403 case ARM::VLD1d8TPseudo:
4404 case ARM::VLD1d16TPseudo:
4405 case ARM::VLD1d32TPseudo:
4406 case ARM::VLD1d64TPseudo:
4407 case ARM::VLD1d64TPseudoWB_fixed:
4408 case ARM::VLD1d64TPseudoWB_register:
4409 case ARM::VLD3d8Pseudo_UPD:
4410 case ARM::VLD3d16Pseudo_UPD:
4411 case ARM::VLD3d32Pseudo_UPD:
4412 case ARM::VLD3q8Pseudo_UPD:
4413 case ARM::VLD3q16Pseudo_UPD:
4414 case ARM::VLD3q32Pseudo_UPD:
4415 case ARM::VLD3q8oddPseudo:
4416 case ARM::VLD3q16oddPseudo:
4417 case ARM::VLD3q32oddPseudo:
4418 case ARM::VLD3q8oddPseudo_UPD:
4419 case ARM::VLD3q16oddPseudo_UPD:
4420 case ARM::VLD3q32oddPseudo_UPD:
4421 case ARM::VLD4d8Pseudo:
4422 case ARM::VLD4d16Pseudo:
4423 case ARM::VLD4d32Pseudo:
4424 case ARM::VLD1d8QPseudo:
4425 case ARM::VLD1d16QPseudo:
4426 case ARM::VLD1d32QPseudo:
4427 case ARM::VLD1d64QPseudo:
4428 case ARM::VLD1d64QPseudoWB_fixed:
4429 case ARM::VLD1d64QPseudoWB_register:
4430 case ARM::VLD1q8HighQPseudo:
4431 case ARM::VLD1q8LowQPseudo_UPD:
4432 case ARM::VLD1q8HighTPseudo:
4433 case ARM::VLD1q8LowTPseudo_UPD:
4434 case ARM::VLD1q16HighQPseudo:
4435 case ARM::VLD1q16LowQPseudo_UPD:
4436 case ARM::VLD1q16HighTPseudo:
4437 case ARM::VLD1q16LowTPseudo_UPD:
4438 case ARM::VLD1q32HighQPseudo:
4439 case ARM::VLD1q32LowQPseudo_UPD:
4440 case ARM::VLD1q32HighTPseudo:
4441 case ARM::VLD1q32LowTPseudo_UPD:
4442 case ARM::VLD1q64HighQPseudo:
4443 case ARM::VLD1q64LowQPseudo_UPD:
4444 case ARM::VLD1q64HighTPseudo:
4445 case ARM::VLD1q64LowTPseudo_UPD:
4446 case ARM::VLD4d8Pseudo_UPD:
4447 case ARM::VLD4d16Pseudo_UPD:
4448 case ARM::VLD4d32Pseudo_UPD:
4449 case ARM::VLD4q8Pseudo_UPD:
4450 case ARM::VLD4q16Pseudo_UPD:
4451 case ARM::VLD4q32Pseudo_UPD:
4452 case ARM::VLD4q8oddPseudo:
4453 case ARM::VLD4q16oddPseudo:
4454 case ARM::VLD4q32oddPseudo:
4455 case ARM::VLD4q8oddPseudo_UPD:
4456 case ARM::VLD4q16oddPseudo_UPD:
4457 case ARM::VLD4q32oddPseudo_UPD:
4458 case ARM::VLD1DUPq8:
4459 case ARM::VLD1DUPq16:
4460 case ARM::VLD1DUPq32:
4461 case ARM::VLD1DUPq8wb_fixed:
4462 case ARM::VLD1DUPq16wb_fixed:
4463 case ARM::VLD1DUPq32wb_fixed:
4464 case ARM::VLD1DUPq8wb_register:
4465 case ARM::VLD1DUPq16wb_register:
4466 case ARM::VLD1DUPq32wb_register:
4467 case ARM::VLD2DUPd8:
4468 case ARM::VLD2DUPd16:
4469 case ARM::VLD2DUPd32:
4470 case ARM::VLD2DUPd8wb_fixed:
4471 case ARM::VLD2DUPd16wb_fixed:
4472 case ARM::VLD2DUPd32wb_fixed:
4473 case ARM::VLD2DUPd8wb_register:
4474 case ARM::VLD2DUPd16wb_register:
4475 case ARM::VLD2DUPd32wb_register:
4476 case ARM::VLD2DUPq8EvenPseudo:
4477 case ARM::VLD2DUPq8OddPseudo:
4478 case ARM::VLD2DUPq16EvenPseudo:
4479 case ARM::VLD2DUPq16OddPseudo:
4480 case ARM::VLD2DUPq32EvenPseudo:
4481 case ARM::VLD2DUPq32OddPseudo:
4482 case ARM::VLD3DUPq8EvenPseudo:
4483 case ARM::VLD3DUPq8OddPseudo:
4484 case ARM::VLD3DUPq16EvenPseudo:
4485 case ARM::VLD3DUPq16OddPseudo:
4486 case ARM::VLD3DUPq32EvenPseudo:
4487 case ARM::VLD3DUPq32OddPseudo:
4488 case ARM::VLD4DUPd8Pseudo:
4489 case ARM::VLD4DUPd16Pseudo:
4490 case ARM::VLD4DUPd32Pseudo:
4491 case ARM::VLD4DUPd8Pseudo_UPD:
4492 case ARM::VLD4DUPd16Pseudo_UPD:
4493 case ARM::VLD4DUPd32Pseudo_UPD:
4494 case ARM::VLD4DUPq8EvenPseudo:
4495 case ARM::VLD4DUPq8OddPseudo:
4496 case ARM::VLD4DUPq16EvenPseudo:
4497 case ARM::VLD4DUPq16OddPseudo:
4498 case ARM::VLD4DUPq32EvenPseudo:
4499 case ARM::VLD4DUPq32OddPseudo:
4500 case ARM::VLD1LNq8Pseudo:
4501 case ARM::VLD1LNq16Pseudo:
4502 case ARM::VLD1LNq32Pseudo:
4503 case ARM::VLD1LNq8Pseudo_UPD:
4504 case ARM::VLD1LNq16Pseudo_UPD:
4505 case ARM::VLD1LNq32Pseudo_UPD:
4506 case ARM::VLD2LNd8Pseudo:
4507 case ARM::VLD2LNd16Pseudo:
4508 case ARM::VLD2LNd32Pseudo:
4509 case ARM::VLD2LNq16Pseudo:
4510 case ARM::VLD2LNq32Pseudo:
4511 case ARM::VLD2LNd8Pseudo_UPD:
4512 case ARM::VLD2LNd16Pseudo_UPD:
4513 case ARM::VLD2LNd32Pseudo_UPD:
4514 case ARM::VLD2LNq16Pseudo_UPD:
4515 case ARM::VLD2LNq32Pseudo_UPD:
4516 case ARM::VLD4LNd8Pseudo:
4517 case ARM::VLD4LNd16Pseudo:
4518 case ARM::VLD4LNd32Pseudo:
4519 case ARM::VLD4LNq16Pseudo:
4520 case ARM::VLD4LNq32Pseudo:
4521 case ARM::VLD4LNd8Pseudo_UPD:
4522 case ARM::VLD4LNd16Pseudo_UPD:
4523 case ARM::VLD4LNd32Pseudo_UPD:
4524 case ARM::VLD4LNq16Pseudo_UPD:
4525 case ARM::VLD4LNq32Pseudo_UPD:
4526 // If the address is not 64-bit aligned, the latencies of these
4527 // instructions increases by one.
4528 Latency = *Latency + 1;
4529 break;
4530 }
4531
4532 return Latency;
4533}
4534
4535unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4536 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4537 MI.isImplicitDef())
4538 return 0;
4539
4540 if (MI.isBundle())
4541 return 0;
4542
4543 const MCInstrDesc &MCID = MI.getDesc();
4544
4545 if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(Reg: ARM::CPSR) &&
4546 !Subtarget.cheapPredicableCPSRDef())) {
4547 // When predicated, CPSR is an additional source operand for CPSR updating
4548 // instructions, this apparently increases their latencies.
4549 return 1;
4550 }
4551 return 0;
4552}
4553
4554unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4555 const MachineInstr &MI,
4556 unsigned *PredCost) const {
4557 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4558 MI.isImplicitDef())
4559 return 1;
4560
4561 // An instruction scheduler typically runs on unbundled instructions, however
4562 // other passes may query the latency of a bundled instruction.
4563 if (MI.isBundle()) {
4564 unsigned Latency = 0;
4565 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4566 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4567 while (++I != E && I->isInsideBundle()) {
4568 if (I->getOpcode() != ARM::t2IT)
4569 Latency += getInstrLatency(ItinData, MI: *I, PredCost);
4570 }
4571 return Latency;
4572 }
4573
4574 const MCInstrDesc &MCID = MI.getDesc();
4575 if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(Reg: ARM::CPSR) &&
4576 !Subtarget.cheapPredicableCPSRDef()))) {
4577 // When predicated, CPSR is an additional source operand for CPSR updating
4578 // instructions, this apparently increases their latencies.
4579 *PredCost = 1;
4580 }
4581 // Be sure to call getStageLatency for an empty itinerary in case it has a
4582 // valid MinLatency property.
4583 if (!ItinData)
4584 return MI.mayLoad() ? 3 : 1;
4585
4586 unsigned Class = MCID.getSchedClass();
4587
4588 // For instructions with variable uops, use uops as latency.
4589 if (!ItinData->isEmpty() && ItinData->getNumMicroOps(ItinClassIndx: Class) < 0)
4590 return getNumMicroOps(ItinData, MI);
4591
4592 // For the common case, fall back on the itinerary's latency.
4593 unsigned Latency = ItinData->getStageLatency(ItinClassIndx: Class);
4594
4595 // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4596 unsigned DefAlign =
4597 MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4598 int Adj = adjustDefLatency(Subtarget, DefMI: MI, DefMCID: MCID, DefAlign);
4599 if (Adj >= 0 || (int)Latency > -Adj) {
4600 return Latency + Adj;
4601 }
4602 return Latency;
4603}
4604
4605unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4606 SDNode *Node) const {
4607 if (!Node->isMachineOpcode())
4608 return 1;
4609
4610 if (!ItinData || ItinData->isEmpty())
4611 return 1;
4612
4613 unsigned Opcode = Node->getMachineOpcode();
4614 switch (Opcode) {
4615 default:
4616 return ItinData->getStageLatency(ItinClassIndx: get(Opcode).getSchedClass());
4617 case ARM::VLDMQIA:
4618 case ARM::VSTMQIA:
4619 return 2;
4620 }
4621}
4622
4623bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4624 const MachineRegisterInfo *MRI,
4625 const MachineInstr &DefMI,
4626 unsigned DefIdx,
4627 const MachineInstr &UseMI,
4628 unsigned UseIdx) const {
4629 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4630 unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4631 if (Subtarget.nonpipelinedVFP() &&
4632 (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4633 return true;
4634
4635 // Hoist VFP / NEON instructions with 4 or higher latency.
4636 unsigned Latency =
4637 SchedModel.computeOperandLatency(DefMI: &DefMI, DefOperIdx: DefIdx, UseMI: &UseMI, UseOperIdx: UseIdx);
4638 if (Latency <= 3)
4639 return false;
4640 return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4641 UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4642}
4643
4644bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4645 const MachineInstr &DefMI,
4646 unsigned DefIdx) const {
4647 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4648 if (!ItinData || ItinData->isEmpty())
4649 return false;
4650
4651 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4652 if (DDomain == ARMII::DomainGeneral) {
4653 unsigned DefClass = DefMI.getDesc().getSchedClass();
4654 std::optional<unsigned> DefCycle =
4655 ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
4656 return DefCycle && DefCycle <= 2U;
4657 }
4658 return false;
4659}
4660
4661bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4662 StringRef &ErrInfo) const {
4663 if (convertAddSubFlagsOpcode(OldOpc: MI.getOpcode())) {
4664 ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4665 return false;
4666 }
4667 if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4668 // Make sure we don't generate a lo-lo mov that isn't supported.
4669 if (!ARM::hGPRRegClass.contains(Reg: MI.getOperand(i: 0).getReg()) &&
4670 !ARM::hGPRRegClass.contains(Reg: MI.getOperand(i: 1).getReg())) {
4671 ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4672 return false;
4673 }
4674 }
4675 if (MI.getOpcode() == ARM::tPUSH ||
4676 MI.getOpcode() == ARM::tPOP ||
4677 MI.getOpcode() == ARM::tPOP_RET) {
4678 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands(), N: 2)) {
4679 if (MO.isImplicit() || !MO.isReg())
4680 continue;
4681 Register Reg = MO.getReg();
4682 if (Reg < ARM::R0 || Reg > ARM::R7) {
4683 if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4684 !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4685 ErrInfo = "Unsupported register in Thumb1 push/pop";
4686 return false;
4687 }
4688 }
4689 }
4690 }
4691 if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
4692 assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
4693 if ((MI.getOperand(i: 4).getImm() != 2 && MI.getOperand(i: 4).getImm() != 3) ||
4694 MI.getOperand(i: 4).getImm() != MI.getOperand(i: 5).getImm() + 2) {
4695 ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
4696 return false;
4697 }
4698 }
4699
4700 // Check the address model by taking the first Imm operand and checking it is
4701 // legal for that addressing mode.
4702 ARMII::AddrMode AddrMode =
4703 (ARMII::AddrMode)(MI.getDesc().TSFlags & ARMII::AddrModeMask);
4704 switch (AddrMode) {
4705 default:
4706 break;
4707 case ARMII::AddrModeT2_i7:
4708 case ARMII::AddrModeT2_i7s2:
4709 case ARMII::AddrModeT2_i7s4:
4710 case ARMII::AddrModeT2_i8:
4711 case ARMII::AddrModeT2_i8pos:
4712 case ARMII::AddrModeT2_i8neg:
4713 case ARMII::AddrModeT2_i8s4:
4714 case ARMII::AddrModeT2_i12: {
4715 uint32_t Imm = 0;
4716 for (auto Op : MI.operands()) {
4717 if (Op.isImm()) {
4718 Imm = Op.getImm();
4719 break;
4720 }
4721 }
4722 if (!isLegalAddressImm(Opcode: MI.getOpcode(), Imm, TII: this)) {
4723 ErrInfo = "Incorrect AddrMode Imm for instruction";
4724 return false;
4725 }
4726 break;
4727 }
4728 }
4729 return true;
4730}
4731
4732void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4733 unsigned LoadImmOpc,
4734 unsigned LoadOpc) const {
4735 assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4736 "ROPI/RWPI not currently supported with stack guard");
4737
4738 MachineBasicBlock &MBB = *MI->getParent();
4739 DebugLoc DL = MI->getDebugLoc();
4740 Register Reg = MI->getOperand(i: 0).getReg();
4741 MachineInstrBuilder MIB;
4742 unsigned int Offset = 0;
4743
4744 if (LoadImmOpc == ARM::MRC || LoadImmOpc == ARM::t2MRC) {
4745 assert(!Subtarget.isReadTPSoft() &&
4746 "TLS stack protector requires hardware TLS register");
4747
4748 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4749 .addImm(Val: 15)
4750 .addImm(Val: 0)
4751 .addImm(Val: 13)
4752 .addImm(Val: 0)
4753 .addImm(Val: 3)
4754 .add(MOs: predOps(Pred: ARMCC::AL));
4755
4756 Module &M = *MBB.getParent()->getFunction().getParent();
4757 Offset = M.getStackProtectorGuardOffset();
4758 if (Offset & ~0xfffU) {
4759 // The offset won't fit in the LDR's 12-bit immediate field, so emit an
4760 // extra ADD to cover the delta. This gives us a guaranteed 8 additional
4761 // bits, resulting in a range of 0 to +1 MiB for the guard offset.
4762 unsigned AddOpc = (LoadImmOpc == ARM::MRC) ? ARM::ADDri : ARM::t2ADDri;
4763 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AddOpc), DestReg: Reg)
4764 .addReg(RegNo: Reg, Flags: RegState::Kill)
4765 .addImm(Val: Offset & ~0xfffU)
4766 .add(MOs: predOps(Pred: ARMCC::AL))
4767 .addReg(RegNo: 0);
4768 Offset &= 0xfffU;
4769 }
4770 } else {
4771 const GlobalValue *GV =
4772 cast<GlobalValue>(Val: (*MI->memoperands_begin())->getValue());
4773 bool IsIndirect = Subtarget.isGVIndirectSymbol(GV);
4774
4775 unsigned TargetFlags = ARMII::MO_NO_FLAG;
4776 if (Subtarget.isTargetMachO()) {
4777 TargetFlags |= ARMII::MO_NONLAZY;
4778 } else if (Subtarget.isTargetCOFF()) {
4779 if (GV->hasDLLImportStorageClass())
4780 TargetFlags |= ARMII::MO_DLLIMPORT;
4781 else if (IsIndirect)
4782 TargetFlags |= ARMII::MO_COFFSTUB;
4783 } else if (IsIndirect) {
4784 TargetFlags |= ARMII::MO_GOT;
4785 }
4786
4787 if (LoadImmOpc == ARM::tMOVi32imm) { // Thumb-1 execute-only
4788 Register CPSRSaveReg = ARM::R12; // Use R12 as scratch register
4789 auto APSREncoding =
4790 ARMSysReg::lookupMClassSysRegByName(Name: "apsr_nzcvq")->Encoding;
4791 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ARM::t2MRS_M), DestReg: CPSRSaveReg)
4792 .addImm(Val: APSREncoding)
4793 .add(MOs: predOps(Pred: ARMCC::AL));
4794 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4795 .addGlobalAddress(GV, Offset: 0, TargetFlags);
4796 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ARM::t2MSR_M))
4797 .addImm(Val: APSREncoding)
4798 .addReg(RegNo: CPSRSaveReg, Flags: RegState::Kill)
4799 .add(MOs: predOps(Pred: ARMCC::AL));
4800 } else {
4801 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4802 .addGlobalAddress(GV, Offset: 0, TargetFlags);
4803 }
4804
4805 if (IsIndirect) {
4806 MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadOpc), DestReg: Reg);
4807 MIB.addReg(RegNo: Reg, Flags: RegState::Kill).addImm(Val: 0);
4808 auto Flags = MachineMemOperand::MOLoad |
4809 MachineMemOperand::MODereferenceable |
4810 MachineMemOperand::MOInvariant;
4811 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4812 PtrInfo: MachinePointerInfo::getGOT(MF&: *MBB.getParent()), F: Flags, Size: 4, BaseAlignment: Align(4));
4813 MIB.addMemOperand(MMO).add(MOs: predOps(Pred: ARMCC::AL));
4814 }
4815 }
4816
4817 MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadOpc), DestReg: Reg);
4818 MIB.addReg(RegNo: Reg, Flags: RegState::Kill)
4819 .addImm(Val: Offset)
4820 .cloneMemRefs(OtherMI: *MI)
4821 .add(MOs: predOps(Pred: ARMCC::AL));
4822}
4823
4824bool
4825ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4826 unsigned &AddSubOpc,
4827 bool &NegAcc, bool &HasLane) const {
4828 auto I = MLxEntryMap.find(Val: Opcode);
4829 if (I == MLxEntryMap.end())
4830 return false;
4831
4832 const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4833 MulOpc = Entry.MulOpc;
4834 AddSubOpc = Entry.AddSubOpc;
4835 NegAcc = Entry.NegAcc;
4836 HasLane = Entry.HasLane;
4837 return true;
4838}
4839
4840//===----------------------------------------------------------------------===//
4841// Execution domains.
4842//===----------------------------------------------------------------------===//
4843//
4844// Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4845// and some can go down both. The vmov instructions go down the VFP pipeline,
4846// but they can be changed to vorr equivalents that are executed by the NEON
4847// pipeline.
4848//
4849// We use the following execution domain numbering:
4850//
4851enum ARMExeDomain {
4852 ExeGeneric = 0,
4853 ExeVFP = 1,
4854 ExeNEON = 2
4855};
4856
4857//
4858// Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4859//
4860std::pair<uint16_t, uint16_t>
4861ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
4862 // If we don't have access to NEON instructions then we won't be able
4863 // to swizzle anything to the NEON domain. Check to make sure.
4864 if (Subtarget.hasNEON()) {
4865 // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4866 // if they are not predicated.
4867 if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
4868 return std::make_pair(x: ExeVFP, y: (1 << ExeVFP) | (1 << ExeNEON));
4869
4870 // CortexA9 is particularly picky about mixing the two and wants these
4871 // converted.
4872 if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
4873 (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
4874 MI.getOpcode() == ARM::VMOVS))
4875 return std::make_pair(x: ExeVFP, y: (1 << ExeVFP) | (1 << ExeNEON));
4876 }
4877 // No other instructions can be swizzled, so just determine their domain.
4878 unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
4879
4880 if (Domain & ARMII::DomainNEON)
4881 return std::make_pair(x: ExeNEON, y: 0);
4882
4883 // Certain instructions can go either way on Cortex-A8.
4884 // Treat them as NEON instructions.
4885 if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
4886 return std::make_pair(x: ExeNEON, y: 0);
4887
4888 if (Domain & ARMII::DomainVFP)
4889 return std::make_pair(x: ExeVFP, y: 0);
4890
4891 return std::make_pair(x: ExeGeneric, y: 0);
4892}
4893
4894static MCRegister getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
4895 unsigned SReg, unsigned &Lane) {
4896 MCRegister DReg =
4897 TRI->getMatchingSuperReg(Reg: SReg, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
4898 Lane = 0;
4899
4900 if (DReg)
4901 return DReg;
4902
4903 Lane = 1;
4904 DReg = TRI->getMatchingSuperReg(Reg: SReg, SubIdx: ARM::ssub_1, RC: &ARM::DPRRegClass);
4905
4906 assert(DReg && "S-register with no D super-register?");
4907 return DReg;
4908}
4909
4910/// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
4911/// set ImplicitSReg to a register number that must be marked as implicit-use or
4912/// zero if no register needs to be defined as implicit-use.
4913///
4914/// If the function cannot determine if an SPR should be marked implicit use or
4915/// not, it returns false.
4916///
4917/// This function handles cases where an instruction is being modified from taking
4918/// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
4919/// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
4920/// lane of the DPR).
4921///
4922/// If the other SPR is defined, an implicit-use of it should be added. Else,
4923/// (including the case where the DPR itself is defined), it should not.
4924///
4925static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
4926 MachineInstr &MI, MCRegister DReg,
4927 unsigned Lane,
4928 MCRegister &ImplicitSReg) {
4929 // If the DPR is defined or used already, the other SPR lane will be chained
4930 // correctly, so there is nothing to be done.
4931 if (MI.definesRegister(Reg: DReg, TRI) || MI.readsRegister(Reg: DReg, TRI)) {
4932 ImplicitSReg = MCRegister();
4933 return true;
4934 }
4935
4936 // Otherwise we need to go searching to see if the SPR is set explicitly.
4937 ImplicitSReg = TRI->getSubReg(Reg: DReg,
4938 Idx: (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
4939 MachineBasicBlock::LivenessQueryResult LQR =
4940 MI.getParent()->computeRegisterLiveness(TRI, Reg: ImplicitSReg, Before: MI);
4941
4942 if (LQR == MachineBasicBlock::LQR_Live)
4943 return true;
4944 else if (LQR == MachineBasicBlock::LQR_Unknown)
4945 return false;
4946
4947 // If the register is known not to be live, there is no need to add an
4948 // implicit-use.
4949 ImplicitSReg = MCRegister();
4950 return true;
4951}
4952
4953void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
4954 unsigned Domain) const {
4955 unsigned DstReg, SrcReg;
4956 MCRegister DReg;
4957 unsigned Lane;
4958 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
4959 const TargetRegisterInfo *TRI = &getRegisterInfo();
4960 switch (MI.getOpcode()) {
4961 default:
4962 llvm_unreachable("cannot handle opcode!");
4963 break;
4964 case ARM::VMOVD:
4965 if (Domain != ExeNEON)
4966 break;
4967
4968 // Zap the predicate operands.
4969 assert(!isPredicated(MI) && "Cannot predicate a VORRd");
4970
4971 // Make sure we've got NEON instructions.
4972 assert(Subtarget.hasNEON() && "VORRd requires NEON");
4973
4974 // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
4975 DstReg = MI.getOperand(i: 0).getReg();
4976 SrcReg = MI.getOperand(i: 1).getReg();
4977
4978 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
4979 MI.removeOperand(OpNo: i - 1);
4980
4981 // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
4982 MI.setDesc(get(Opcode: ARM::VORRd));
4983 MIB.addReg(RegNo: DstReg, Flags: RegState::Define)
4984 .addReg(RegNo: SrcReg)
4985 .addReg(RegNo: SrcReg)
4986 .add(MOs: predOps(Pred: ARMCC::AL));
4987 break;
4988 case ARM::VMOVRS:
4989 if (Domain != ExeNEON)
4990 break;
4991 assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
4992
4993 // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
4994 DstReg = MI.getOperand(i: 0).getReg();
4995 SrcReg = MI.getOperand(i: 1).getReg();
4996
4997 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
4998 MI.removeOperand(OpNo: i - 1);
4999
5000 DReg = getCorrespondingDRegAndLane(TRI, SReg: SrcReg, Lane);
5001
5002 // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5003 // Note that DSrc has been widened and the other lane may be undef, which
5004 // contaminates the entire register.
5005 MI.setDesc(get(Opcode: ARM::VGETLNi32));
5006 MIB.addReg(RegNo: DstReg, Flags: RegState::Define)
5007 .addReg(RegNo: DReg, Flags: RegState::Undef)
5008 .addImm(Val: Lane)
5009 .add(MOs: predOps(Pred: ARMCC::AL));
5010
5011 // The old source should be an implicit use, otherwise we might think it
5012 // was dead before here.
5013 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5014 break;
5015 case ARM::VMOVSR: {
5016 if (Domain != ExeNEON)
5017 break;
5018 assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5019
5020 // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5021 DstReg = MI.getOperand(i: 0).getReg();
5022 SrcReg = MI.getOperand(i: 1).getReg();
5023
5024 DReg = getCorrespondingDRegAndLane(TRI, SReg: DstReg, Lane);
5025
5026 MCRegister ImplicitSReg;
5027 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5028 break;
5029
5030 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5031 MI.removeOperand(OpNo: i - 1);
5032
5033 // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5034 // Again DDst may be undefined at the beginning of this instruction.
5035 MI.setDesc(get(Opcode: ARM::VSETLNi32));
5036 MIB.addReg(RegNo: DReg, Flags: RegState::Define)
5037 .addReg(RegNo: DReg, Flags: getUndefRegState(B: !MI.readsRegister(Reg: DReg, TRI)))
5038 .addReg(RegNo: SrcReg)
5039 .addImm(Val: Lane)
5040 .add(MOs: predOps(Pred: ARMCC::AL));
5041
5042 // The narrower destination must be marked as set to keep previous chains
5043 // in place.
5044 MIB.addReg(RegNo: DstReg, Flags: RegState::Define | RegState::Implicit);
5045 if (ImplicitSReg)
5046 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5047 break;
5048 }
5049 case ARM::VMOVS: {
5050 if (Domain != ExeNEON)
5051 break;
5052
5053 // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5054 DstReg = MI.getOperand(i: 0).getReg();
5055 SrcReg = MI.getOperand(i: 1).getReg();
5056
5057 unsigned DstLane = 0, SrcLane = 0;
5058 MCRegister DDst, DSrc;
5059 DDst = getCorrespondingDRegAndLane(TRI, SReg: DstReg, Lane&: DstLane);
5060 DSrc = getCorrespondingDRegAndLane(TRI, SReg: SrcReg, Lane&: SrcLane);
5061
5062 MCRegister ImplicitSReg;
5063 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg: DSrc, Lane: SrcLane, ImplicitSReg))
5064 break;
5065
5066 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5067 MI.removeOperand(OpNo: i - 1);
5068
5069 if (DSrc == DDst) {
5070 // Destination can be:
5071 // %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5072 MI.setDesc(get(Opcode: ARM::VDUPLN32d));
5073 MIB.addReg(RegNo: DDst, Flags: RegState::Define)
5074 .addReg(RegNo: DDst, Flags: getUndefRegState(B: !MI.readsRegister(Reg: DDst, TRI)))
5075 .addImm(Val: SrcLane)
5076 .add(MOs: predOps(Pred: ARMCC::AL));
5077
5078 // Neither the source or the destination are naturally represented any
5079 // more, so add them in manually.
5080 MIB.addReg(RegNo: DstReg, Flags: RegState::Implicit | RegState::Define);
5081 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5082 if (ImplicitSReg)
5083 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5084 break;
5085 }
5086
5087 // In general there's no single instruction that can perform an S <-> S
5088 // move in NEON space, but a pair of VEXT instructions *can* do the
5089 // job. It turns out that the VEXTs needed will only use DSrc once, with
5090 // the position based purely on the combination of lane-0 and lane-1
5091 // involved. For example
5092 // vmov s0, s2 -> vext.32 d0, d0, d1, #1 vext.32 d0, d0, d0, #1
5093 // vmov s1, s3 -> vext.32 d0, d1, d0, #1 vext.32 d0, d0, d0, #1
5094 // vmov s0, s3 -> vext.32 d0, d0, d0, #1 vext.32 d0, d1, d0, #1
5095 // vmov s1, s2 -> vext.32 d0, d0, d0, #1 vext.32 d0, d0, d1, #1
5096 //
5097 // Pattern of the MachineInstrs is:
5098 // %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5099 MachineInstrBuilder NewMIB;
5100 NewMIB = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: ARM::VEXTd32),
5101 DestReg: DDst);
5102
5103 // On the first instruction, both DSrc and DDst may be undef if present.
5104 // Specifically when the original instruction didn't have them as an
5105 // <imp-use>.
5106 MCRegister CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5107 bool CurUndef = !MI.readsRegister(Reg: CurReg, TRI);
5108 NewMIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef));
5109
5110 CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5111 CurUndef = !MI.readsRegister(Reg: CurReg, TRI);
5112 NewMIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef))
5113 .addImm(Val: 1)
5114 .add(MOs: predOps(Pred: ARMCC::AL));
5115
5116 if (SrcLane == DstLane)
5117 NewMIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5118
5119 MI.setDesc(get(Opcode: ARM::VEXTd32));
5120 MIB.addReg(RegNo: DDst, Flags: RegState::Define);
5121
5122 // On the second instruction, DDst has definitely been defined above, so
5123 // it is not undef. DSrc, if present, can be undef as above.
5124 CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5125 CurUndef = CurReg == DSrc && !MI.readsRegister(Reg: CurReg, TRI);
5126 MIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef));
5127
5128 CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5129 CurUndef = CurReg == DSrc && !MI.readsRegister(Reg: CurReg, TRI);
5130 MIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef))
5131 .addImm(Val: 1)
5132 .add(MOs: predOps(Pred: ARMCC::AL));
5133
5134 if (SrcLane != DstLane)
5135 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5136
5137 // As before, the original destination is no longer represented, add it
5138 // implicitly.
5139 MIB.addReg(RegNo: DstReg, Flags: RegState::Define | RegState::Implicit);
5140 if (ImplicitSReg != 0)
5141 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5142 break;
5143 }
5144 }
5145}
5146
5147//===----------------------------------------------------------------------===//
5148// Partial register updates
5149//===----------------------------------------------------------------------===//
5150//
5151// Swift renames NEON registers with 64-bit granularity. That means any
5152// instruction writing an S-reg implicitly reads the containing D-reg. The
5153// problem is mostly avoided by translating f32 operations to v2f32 operations
5154// on D-registers, but f32 loads are still a problem.
5155//
5156// These instructions can load an f32 into a NEON register:
5157//
5158// VLDRS - Only writes S, partial D update.
5159// VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5160// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5161//
5162// FCONSTD can be used as a dependency-breaking instruction.
5163unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5164 const MachineInstr &MI, unsigned OpNum,
5165 const TargetRegisterInfo *TRI) const {
5166 auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5167 if (!PartialUpdateClearance)
5168 return 0;
5169
5170 assert(TRI && "Need TRI instance");
5171
5172 const MachineOperand &MO = MI.getOperand(i: OpNum);
5173 if (MO.readsReg())
5174 return 0;
5175 Register Reg = MO.getReg();
5176 int UseOp = -1;
5177
5178 switch (MI.getOpcode()) {
5179 // Normal instructions writing only an S-register.
5180 case ARM::VLDRS:
5181 case ARM::FCONSTS:
5182 case ARM::VMOVSR:
5183 case ARM::VMOVv8i8:
5184 case ARM::VMOVv4i16:
5185 case ARM::VMOVv2i32:
5186 case ARM::VMOVv2f32:
5187 case ARM::VMOVv1i64:
5188 UseOp = MI.findRegisterUseOperandIdx(Reg, TRI, isKill: false);
5189 break;
5190
5191 // Explicitly reads the dependency.
5192 case ARM::VLD1LNd32:
5193 UseOp = 3;
5194 break;
5195 default:
5196 return 0;
5197 }
5198
5199 // If this instruction actually reads a value from Reg, there is no unwanted
5200 // dependency.
5201 if (UseOp != -1 && MI.getOperand(i: UseOp).readsReg())
5202 return 0;
5203
5204 // We must be able to clobber the whole D-reg.
5205 if (Reg.isVirtual()) {
5206 // Virtual register must be a def undef foo:ssub_0 operand.
5207 if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5208 return 0;
5209 } else if (ARM::SPRRegClass.contains(Reg)) {
5210 // Physical register: MI must define the full D-reg.
5211 MCRegister DReg =
5212 TRI->getMatchingSuperReg(Reg, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
5213 if (!DReg || !MI.definesRegister(Reg: DReg, TRI))
5214 return 0;
5215 }
5216
5217 // MI has an unwanted D-register dependency.
5218 // Avoid defs in the previous N instructrions.
5219 return PartialUpdateClearance;
5220}
5221
5222// Break a partial register dependency after getPartialRegUpdateClearance
5223// returned non-zero.
5224void ARMBaseInstrInfo::breakPartialRegDependency(
5225 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5226 assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5227 assert(TRI && "Need TRI instance");
5228
5229 const MachineOperand &MO = MI.getOperand(i: OpNum);
5230 Register Reg = MO.getReg();
5231 assert(Reg.isPhysical() && "Can't break virtual register dependencies.");
5232 unsigned DReg = Reg;
5233
5234 // If MI defines an S-reg, find the corresponding D super-register.
5235 if (ARM::SPRRegClass.contains(Reg)) {
5236 DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5237 assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5238 }
5239
5240 assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5241 assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5242
5243 // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5244 // the full D-register by loading the same value to both lanes. The
5245 // instruction is micro-coded with 2 uops, so don't do this until we can
5246 // properly schedule micro-coded instructions. The dispatcher stalls cause
5247 // too big regressions.
5248
5249 // Insert the dependency-breaking FCONSTD before MI.
5250 // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5251 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: ARM::FCONSTD), DestReg: DReg)
5252 .addImm(Val: 96)
5253 .add(MOs: predOps(Pred: ARMCC::AL));
5254 MI.addRegisterKilled(IncomingReg: DReg, RegInfo: TRI, AddIfNotFound: true);
5255}
5256
5257bool ARMBaseInstrInfo::hasNOP() const {
5258 return Subtarget.hasFeature(Feature: ARM::HasV6KOps);
5259}
5260
5261bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5262 if (MI->getNumOperands() < 4)
5263 return true;
5264 unsigned ShOpVal = MI->getOperand(i: 3).getImm();
5265 unsigned ShImm = ARM_AM::getSORegOffset(Op: ShOpVal);
5266 // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5267 if ((ShImm == 1 && ARM_AM::getSORegShOp(Op: ShOpVal) == ARM_AM::lsr) ||
5268 ((ShImm == 1 || ShImm == 2) &&
5269 ARM_AM::getSORegShOp(Op: ShOpVal) == ARM_AM::lsl))
5270 return true;
5271
5272 return false;
5273}
5274
5275bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5276 const MachineInstr &MI, unsigned DefIdx,
5277 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5278 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5279 assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5280
5281 switch (MI.getOpcode()) {
5282 case ARM::VMOVDRR:
5283 // dX = VMOVDRR rY, rZ
5284 // is the same as:
5285 // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5286 // Populate the InputRegs accordingly.
5287 // rY
5288 const MachineOperand *MOReg = &MI.getOperand(i: 1);
5289 if (!MOReg->isUndef())
5290 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg->getReg(),
5291 MOReg->getSubReg(), ARM::ssub_0));
5292 // rZ
5293 MOReg = &MI.getOperand(i: 2);
5294 if (!MOReg->isUndef())
5295 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg->getReg(),
5296 MOReg->getSubReg(), ARM::ssub_1));
5297 return true;
5298 }
5299 llvm_unreachable("Target dependent opcode missing");
5300}
5301
5302bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5303 const MachineInstr &MI, unsigned DefIdx,
5304 RegSubRegPairAndIdx &InputReg) const {
5305 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5306 assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5307
5308 switch (MI.getOpcode()) {
5309 case ARM::VMOVRRD:
5310 // rX, rY = VMOVRRD dZ
5311 // is the same as:
5312 // rX = EXTRACT_SUBREG dZ, ssub_0
5313 // rY = EXTRACT_SUBREG dZ, ssub_1
5314 const MachineOperand &MOReg = MI.getOperand(i: 2);
5315 if (MOReg.isUndef())
5316 return false;
5317 InputReg.Reg = MOReg.getReg();
5318 InputReg.SubReg = MOReg.getSubReg();
5319 InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5320 return true;
5321 }
5322 llvm_unreachable("Target dependent opcode missing");
5323}
5324
5325bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5326 const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5327 RegSubRegPairAndIdx &InsertedReg) const {
5328 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5329 assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5330
5331 switch (MI.getOpcode()) {
5332 case ARM::VSETLNi32:
5333 case ARM::MVE_VMOV_to_lane_32:
5334 // dX = VSETLNi32 dY, rZ, imm
5335 // qX = MVE_VMOV_to_lane_32 qY, rZ, imm
5336 const MachineOperand &MOBaseReg = MI.getOperand(i: 1);
5337 const MachineOperand &MOInsertedReg = MI.getOperand(i: 2);
5338 if (MOInsertedReg.isUndef())
5339 return false;
5340 const MachineOperand &MOIndex = MI.getOperand(i: 3);
5341 BaseReg.Reg = MOBaseReg.getReg();
5342 BaseReg.SubReg = MOBaseReg.getSubReg();
5343
5344 InsertedReg.Reg = MOInsertedReg.getReg();
5345 InsertedReg.SubReg = MOInsertedReg.getSubReg();
5346 InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm();
5347 return true;
5348 }
5349 llvm_unreachable("Target dependent opcode missing");
5350}
5351
5352std::pair<unsigned, unsigned>
5353ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5354 const unsigned Mask = ARMII::MO_OPTION_MASK;
5355 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
5356}
5357
5358ArrayRef<std::pair<unsigned, const char *>>
5359ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5360 using namespace ARMII;
5361
5362 static const std::pair<unsigned, const char *> TargetFlags[] = {
5363 {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"},
5364 {MO_LO_0_7, "arm-lo-0-7"}, {MO_HI_0_7, "arm-hi-0-7"},
5365 {MO_LO_8_15, "arm-lo-8-15"}, {MO_HI_8_15, "arm-hi-8-15"},
5366 };
5367 return ArrayRef(TargetFlags);
5368}
5369
5370ArrayRef<std::pair<unsigned, const char *>>
5371ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5372 using namespace ARMII;
5373
5374 static const std::pair<unsigned, const char *> TargetFlags[] = {
5375 {MO_COFFSTUB, "arm-coffstub"},
5376 {MO_GOT, "arm-got"},
5377 {MO_SBREL, "arm-sbrel"},
5378 {MO_DLLIMPORT, "arm-dllimport"},
5379 {MO_SECREL, "arm-secrel"},
5380 {MO_NONLAZY, "arm-nonlazy"}};
5381 return ArrayRef(TargetFlags);
5382}
5383
5384std::optional<RegImmPair>
5385ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
5386 int Sign = 1;
5387 unsigned Opcode = MI.getOpcode();
5388 int64_t Offset = 0;
5389
5390 // TODO: Handle cases where Reg is a super- or sub-register of the
5391 // destination register.
5392 const MachineOperand &Op0 = MI.getOperand(i: 0);
5393 if (!Op0.isReg() || Reg != Op0.getReg())
5394 return std::nullopt;
5395
5396 // We describe SUBri or ADDri instructions.
5397 if (Opcode == ARM::SUBri)
5398 Sign = -1;
5399 else if (Opcode != ARM::ADDri)
5400 return std::nullopt;
5401
5402 // TODO: Third operand can be global address (usually some string). Since
5403 // strings can be relocated we cannot calculate their offsets for
5404 // now.
5405 if (!MI.getOperand(i: 1).isReg() || !MI.getOperand(i: 2).isImm())
5406 return std::nullopt;
5407
5408 Offset = MI.getOperand(i: 2).getImm() * Sign;
5409 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
5410}
5411
5412bool llvm::registerDefinedBetween(unsigned Reg,
5413 MachineBasicBlock::iterator From,
5414 MachineBasicBlock::iterator To,
5415 const TargetRegisterInfo *TRI) {
5416 for (auto I = From; I != To; ++I)
5417 if (I->modifiesRegister(Reg, TRI))
5418 return true;
5419 return false;
5420}
5421
5422MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5423 const TargetRegisterInfo *TRI) {
5424 // Search backwards to the instruction that defines CSPR. This may or not
5425 // be a CMP, we check that after this loop. If we find another instruction
5426 // that reads cpsr, we return nullptr.
5427 MachineBasicBlock::iterator CmpMI = Br;
5428 while (CmpMI != Br->getParent()->begin()) {
5429 --CmpMI;
5430 if (CmpMI->modifiesRegister(Reg: ARM::CPSR, TRI))
5431 break;
5432 if (CmpMI->readsRegister(Reg: ARM::CPSR, TRI))
5433 break;
5434 }
5435
5436 // Check that this inst is a CMP r[0-7], #0 and that the register
5437 // is not redefined between the cmp and the br.
5438 if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5439 return nullptr;
5440 Register Reg = CmpMI->getOperand(i: 0).getReg();
5441 Register PredReg;
5442 ARMCC::CondCodes Pred = getInstrPredicate(MI: *CmpMI, PredReg);
5443 if (Pred != ARMCC::AL || CmpMI->getOperand(i: 1).getImm() != 0)
5444 return nullptr;
5445 if (!isARMLowRegister(Reg))
5446 return nullptr;
5447 if (registerDefinedBetween(Reg, From: CmpMI->getNextNode(), To: Br, TRI))
5448 return nullptr;
5449
5450 return &*CmpMI;
5451}
5452
5453unsigned llvm::ConstantMaterializationCost(unsigned Val,
5454 const ARMSubtarget *Subtarget,
5455 bool ForCodesize) {
5456 if (Subtarget->isThumb()) {
5457 if (Val <= 255) // MOV
5458 return ForCodesize ? 2 : 1;
5459 if (Subtarget->hasV6T2Ops() && (Val <= 0xffff || // MOV
5460 ARM_AM::getT2SOImmVal(Arg: Val) != -1 || // MOVW
5461 ARM_AM::getT2SOImmVal(Arg: ~Val) != -1)) // MVN
5462 return ForCodesize ? 4 : 1;
5463 if (Val <= 510) // MOV + ADDi8
5464 return ForCodesize ? 4 : 2;
5465 if (~Val <= 255) // MOV + MVN
5466 return ForCodesize ? 4 : 2;
5467 if (ARM_AM::isThumbImmShiftedVal(V: Val)) // MOV + LSL
5468 return ForCodesize ? 4 : 2;
5469 } else {
5470 if (ARM_AM::getSOImmVal(Arg: Val) != -1) // MOV
5471 return ForCodesize ? 4 : 1;
5472 if (ARM_AM::getSOImmVal(Arg: ~Val) != -1) // MVN
5473 return ForCodesize ? 4 : 1;
5474 if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5475 return ForCodesize ? 4 : 1;
5476 if (ARM_AM::isSOImmTwoPartVal(V: Val)) // two instrs
5477 return ForCodesize ? 8 : 2;
5478 if (ARM_AM::isSOImmTwoPartValNeg(V: Val)) // two instrs
5479 return ForCodesize ? 8 : 2;
5480 }
5481 if (Subtarget->useMovt()) // MOVW + MOVT
5482 return ForCodesize ? 8 : 2;
5483 return ForCodesize ? 8 : 3; // Literal pool load
5484}
5485
5486bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5487 const ARMSubtarget *Subtarget,
5488 bool ForCodesize) {
5489 // Check with ForCodesize
5490 unsigned Cost1 = ConstantMaterializationCost(Val: Val1, Subtarget, ForCodesize);
5491 unsigned Cost2 = ConstantMaterializationCost(Val: Val2, Subtarget, ForCodesize);
5492 if (Cost1 < Cost2)
5493 return true;
5494 if (Cost1 > Cost2)
5495 return false;
5496
5497 // If they are equal, try with !ForCodesize
5498 return ConstantMaterializationCost(Val: Val1, Subtarget, ForCodesize: !ForCodesize) <
5499 ConstantMaterializationCost(Val: Val2, Subtarget, ForCodesize: !ForCodesize);
5500}
5501
5502/// Constants defining how certain sequences should be outlined.
5503/// This encompasses how an outlined function should be called, and what kind of
5504/// frame should be emitted for that outlined function.
5505///
5506/// \p MachineOutlinerTailCall implies that the function is being created from
5507/// a sequence of instructions ending in a return.
5508///
5509/// That is,
5510///
5511/// I1 OUTLINED_FUNCTION:
5512/// I2 --> B OUTLINED_FUNCTION I1
5513/// BX LR I2
5514/// BX LR
5515///
5516/// +-------------------------+--------+-----+
5517/// | | Thumb2 | ARM |
5518/// +-------------------------+--------+-----+
5519/// | Call overhead in Bytes | 4 | 4 |
5520/// | Frame overhead in Bytes | 0 | 0 |
5521/// | Stack fixup required | No | No |
5522/// +-------------------------+--------+-----+
5523///
5524/// \p MachineOutlinerThunk implies that the function is being created from
5525/// a sequence of instructions ending in a call. The outlined function is
5526/// called with a BL instruction, and the outlined function tail-calls the
5527/// original call destination.
5528///
5529/// That is,
5530///
5531/// I1 OUTLINED_FUNCTION:
5532/// I2 --> BL OUTLINED_FUNCTION I1
5533/// BL f I2
5534/// B f
5535///
5536/// +-------------------------+--------+-----+
5537/// | | Thumb2 | ARM |
5538/// +-------------------------+--------+-----+
5539/// | Call overhead in Bytes | 4 | 4 |
5540/// | Frame overhead in Bytes | 0 | 0 |
5541/// | Stack fixup required | No | No |
5542/// +-------------------------+--------+-----+
5543///
5544/// \p MachineOutlinerNoLRSave implies that the function should be called using
5545/// a BL instruction, but doesn't require LR to be saved and restored. This
5546/// happens when LR is known to be dead.
5547///
5548/// That is,
5549///
5550/// I1 OUTLINED_FUNCTION:
5551/// I2 --> BL OUTLINED_FUNCTION I1
5552/// I3 I2
5553/// I3
5554/// BX LR
5555///
5556/// +-------------------------+--------+-----+
5557/// | | Thumb2 | ARM |
5558/// +-------------------------+--------+-----+
5559/// | Call overhead in Bytes | 4 | 4 |
5560/// | Frame overhead in Bytes | 2 | 4 |
5561/// | Stack fixup required | No | No |
5562/// +-------------------------+--------+-----+
5563///
5564/// \p MachineOutlinerRegSave implies that the function should be called with a
5565/// save and restore of LR to an available register. This allows us to avoid
5566/// stack fixups. Note that this outlining variant is compatible with the
5567/// NoLRSave case.
5568///
5569/// That is,
5570///
5571/// I1 Save LR OUTLINED_FUNCTION:
5572/// I2 --> BL OUTLINED_FUNCTION I1
5573/// I3 Restore LR I2
5574/// I3
5575/// BX LR
5576///
5577/// +-------------------------+--------+-----+
5578/// | | Thumb2 | ARM |
5579/// +-------------------------+--------+-----+
5580/// | Call overhead in Bytes | 8 | 12 |
5581/// | Frame overhead in Bytes | 2 | 4 |
5582/// | Stack fixup required | No | No |
5583/// +-------------------------+--------+-----+
5584///
5585/// \p MachineOutlinerDefault implies that the function should be called with
5586/// a save and restore of LR to the stack.
5587///
5588/// That is,
5589///
5590/// I1 Save LR OUTLINED_FUNCTION:
5591/// I2 --> BL OUTLINED_FUNCTION I1
5592/// I3 Restore LR I2
5593/// I3
5594/// BX LR
5595///
5596/// +-------------------------+--------+-----+
5597/// | | Thumb2 | ARM |
5598/// +-------------------------+--------+-----+
5599/// | Call overhead in Bytes | 8 | 12 |
5600/// | Frame overhead in Bytes | 2 | 4 |
5601/// | Stack fixup required | Yes | Yes |
5602/// +-------------------------+--------+-----+
5603
5604enum MachineOutlinerClass {
5605 MachineOutlinerTailCall,
5606 MachineOutlinerThunk,
5607 MachineOutlinerNoLRSave,
5608 MachineOutlinerRegSave,
5609 MachineOutlinerDefault
5610};
5611
5612enum MachineOutlinerMBBFlags {
5613 LRUnavailableSomewhere = 0x2,
5614 HasCalls = 0x4,
5615 UnsafeRegsDead = 0x8
5616};
5617
5618struct OutlinerCosts {
5619 int CallTailCall;
5620 int FrameTailCall;
5621 int CallThunk;
5622 int FrameThunk;
5623 int CallNoLRSave;
5624 int FrameNoLRSave;
5625 int CallRegSave;
5626 int FrameRegSave;
5627 int CallDefault;
5628 int FrameDefault;
5629 int SaveRestoreLROnStack;
5630
5631 OutlinerCosts(const ARMSubtarget &target)
5632 : CallTailCall(target.isThumb() ? 4 : 4),
5633 FrameTailCall(target.isThumb() ? 0 : 0),
5634 CallThunk(target.isThumb() ? 4 : 4),
5635 FrameThunk(target.isThumb() ? 0 : 0),
5636 CallNoLRSave(target.isThumb() ? 4 : 4),
5637 FrameNoLRSave(target.isThumb() ? 2 : 4),
5638 CallRegSave(target.isThumb() ? 8 : 12),
5639 FrameRegSave(target.isThumb() ? 2 : 4),
5640 CallDefault(target.isThumb() ? 8 : 12),
5641 FrameDefault(target.isThumb() ? 2 : 4),
5642 SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {}
5643};
5644
5645Register
5646ARMBaseInstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
5647 MachineFunction *MF = C.getMF();
5648 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
5649 const ARMBaseRegisterInfo *ARI =
5650 static_cast<const ARMBaseRegisterInfo *>(&TRI);
5651
5652 BitVector regsReserved = ARI->getReservedRegs(MF: *MF);
5653 // Check if there is an available register across the sequence that we can
5654 // use.
5655 for (Register Reg : ARM::rGPRRegClass) {
5656 if (!(Reg < regsReserved.size() && regsReserved.test(Idx: Reg)) &&
5657 Reg != ARM::LR && // LR is not reserved, but don't use it.
5658 Reg != ARM::R12 && // R12 is not guaranteed to be preserved.
5659 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
5660 C.isAvailableInsideSeq(Reg, TRI))
5661 return Reg;
5662 }
5663 return Register();
5664}
5665
5666// Compute liveness of LR at the point after the interval [I, E), which
5667// denotes a *backward* iteration through instructions. Used only for return
5668// basic blocks, which do not end with a tail call.
5669static bool isLRAvailable(const TargetRegisterInfo &TRI,
5670 MachineBasicBlock::reverse_iterator I,
5671 MachineBasicBlock::reverse_iterator E) {
5672 // At the end of the function LR dead.
5673 bool Live = false;
5674 for (; I != E; ++I) {
5675 const MachineInstr &MI = *I;
5676
5677 // Check defs of LR.
5678 if (MI.modifiesRegister(Reg: ARM::LR, TRI: &TRI))
5679 Live = false;
5680
5681 // Check uses of LR.
5682 unsigned Opcode = MI.getOpcode();
5683 if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR ||
5684 Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET ||
5685 Opcode == ARM::tBXNS_RET || Opcode == ARM::t2BXAUT_RET) {
5686 // These instructions use LR, but it's not an (explicit or implicit)
5687 // operand.
5688 Live = true;
5689 continue;
5690 }
5691 if (MI.readsRegister(Reg: ARM::LR, TRI: &TRI))
5692 Live = true;
5693 }
5694 return !Live;
5695}
5696
5697std::optional<std::unique_ptr<outliner::OutlinedFunction>>
5698ARMBaseInstrInfo::getOutliningCandidateInfo(
5699 const MachineModuleInfo &MMI,
5700 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
5701 unsigned MinRepeats) const {
5702 unsigned SequenceSize = 0;
5703 for (auto &MI : RepeatedSequenceLocs[0])
5704 SequenceSize += getInstSizeInBytes(MI);
5705
5706 // Properties about candidate MBBs that hold for all of them.
5707 unsigned FlagsSetInAll = 0xF;
5708
5709 // Compute liveness information for each candidate, and set FlagsSetInAll.
5710 const TargetRegisterInfo &TRI = getRegisterInfo();
5711 for (outliner::Candidate &C : RepeatedSequenceLocs)
5712 FlagsSetInAll &= C.Flags;
5713
5714 // According to the ARM Procedure Call Standard, the following are
5715 // undefined on entry/exit from a function call:
5716 //
5717 // * Register R12(IP),
5718 // * Condition codes (and thus the CPSR register)
5719 //
5720 // Since we control the instructions which are part of the outlined regions
5721 // we don't need to be fully compliant with the AAPCS, but we have to
5722 // guarantee that if a veneer is inserted at link time the code is still
5723 // correct. Because of this, we can't outline any sequence of instructions
5724 // where one of these registers is live into/across it. Thus, we need to
5725 // delete those candidates.
5726 auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5727 // If the unsafe registers in this block are all dead, then we don't need
5728 // to compute liveness here.
5729 if (C.Flags & UnsafeRegsDead)
5730 return false;
5731 return C.isAnyUnavailableAcrossOrOutOfSeq(Regs: {ARM::R12, ARM::CPSR}, TRI);
5732 };
5733
5734 // Are there any candidates where those registers are live?
5735 if (!(FlagsSetInAll & UnsafeRegsDead)) {
5736 // Erase every candidate that violates the restrictions above. (It could be
5737 // true that we have viable candidates, so it's not worth bailing out in
5738 // the case that, say, 1 out of 20 candidates violate the restructions.)
5739 llvm::erase_if(C&: RepeatedSequenceLocs, P: CantGuaranteeValueAcrossCall);
5740
5741 // If the sequence doesn't have enough candidates left, then we're done.
5742 if (RepeatedSequenceLocs.size() < MinRepeats)
5743 return std::nullopt;
5744 }
5745
5746 // We expect the majority of the outlining candidates to be in consensus with
5747 // regard to return address sign and authentication, and branch target
5748 // enforcement, in other words, partitioning according to all the four
5749 // possible combinations of PAC-RET and BTI is going to yield one big subset
5750 // and three small (likely empty) subsets. That allows us to cull incompatible
5751 // candidates separately for PAC-RET and BTI.
5752
5753 // Partition the candidates in two sets: one with BTI enabled and one with BTI
5754 // disabled. Remove the candidates from the smaller set. If they are the same
5755 // number prefer the non-BTI ones for outlining, since they have less
5756 // overhead.
5757 auto NoBTI =
5758 llvm::partition(Range&: RepeatedSequenceLocs, P: [](const outliner::Candidate &C) {
5759 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5760 return AFI.branchTargetEnforcement();
5761 });
5762 if (std::distance(first: RepeatedSequenceLocs.begin(), last: NoBTI) >
5763 std::distance(first: NoBTI, last: RepeatedSequenceLocs.end()))
5764 RepeatedSequenceLocs.erase(first: NoBTI, last: RepeatedSequenceLocs.end());
5765 else
5766 RepeatedSequenceLocs.erase(first: RepeatedSequenceLocs.begin(), last: NoBTI);
5767
5768 if (RepeatedSequenceLocs.size() < MinRepeats)
5769 return std::nullopt;
5770
5771 // Likewise, partition the candidates according to PAC-RET enablement.
5772 auto NoPAC =
5773 llvm::partition(Range&: RepeatedSequenceLocs, P: [](const outliner::Candidate &C) {
5774 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5775 // If the function happens to not spill the LR, do not disqualify it
5776 // from the outlining.
5777 return AFI.shouldSignReturnAddress(SpillsLR: true);
5778 });
5779 if (std::distance(first: RepeatedSequenceLocs.begin(), last: NoPAC) >
5780 std::distance(first: NoPAC, last: RepeatedSequenceLocs.end()))
5781 RepeatedSequenceLocs.erase(first: NoPAC, last: RepeatedSequenceLocs.end());
5782 else
5783 RepeatedSequenceLocs.erase(first: RepeatedSequenceLocs.begin(), last: NoPAC);
5784
5785 if (RepeatedSequenceLocs.size() < MinRepeats)
5786 return std::nullopt;
5787
5788 // At this point, we have only "safe" candidates to outline. Figure out
5789 // frame + call instruction information.
5790
5791 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
5792
5793 // Helper lambda which sets call information for every candidate.
5794 auto SetCandidateCallInfo =
5795 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5796 for (outliner::Candidate &C : RepeatedSequenceLocs)
5797 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
5798 };
5799
5800 OutlinerCosts Costs(Subtarget);
5801
5802 const auto &SomeMFI =
5803 *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>();
5804 // Adjust costs to account for the BTI instructions.
5805 if (SomeMFI.branchTargetEnforcement()) {
5806 Costs.FrameDefault += 4;
5807 Costs.FrameNoLRSave += 4;
5808 Costs.FrameRegSave += 4;
5809 Costs.FrameTailCall += 4;
5810 Costs.FrameThunk += 4;
5811 }
5812
5813 // Adjust costs to account for sign and authentication instructions.
5814 if (SomeMFI.shouldSignReturnAddress(SpillsLR: true)) {
5815 Costs.CallDefault += 8; // +PAC instr, +AUT instr
5816 Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr
5817 }
5818
5819 unsigned FrameID = MachineOutlinerDefault;
5820 unsigned NumBytesToCreateFrame = Costs.FrameDefault;
5821
5822 // If the last instruction in any candidate is a terminator, then we should
5823 // tail call all of the candidates.
5824 if (RepeatedSequenceLocs[0].back().isTerminator()) {
5825 FrameID = MachineOutlinerTailCall;
5826 NumBytesToCreateFrame = Costs.FrameTailCall;
5827 SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall);
5828 } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX ||
5829 LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL ||
5830 LastInstrOpcode == ARM::tBLXi ||
5831 ((LastInstrOpcode == ARM::tBLXr ||
5832 LastInstrOpcode == ARM::tBLXr_noip) &&
5833 ARM::tcGPRRegClass.contains(
5834 Reg: RepeatedSequenceLocs[0].back().getOperand(i: 2).getReg()))) {
5835 FrameID = MachineOutlinerThunk;
5836 NumBytesToCreateFrame = Costs.FrameThunk;
5837 SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk);
5838 } else {
5839 // We need to decide how to emit calls + frames. We can always emit the same
5840 // frame if we don't need to save to the stack. If we have to save to the
5841 // stack, then we need a different frame.
5842 unsigned NumBytesNoStackCalls = 0;
5843 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
5844
5845 for (outliner::Candidate &C : RepeatedSequenceLocs) {
5846 // LR liveness is overestimated in return blocks, unless they end with a
5847 // tail call.
5848 const auto Last = C.getMBB()->rbegin();
5849 const bool LRIsAvailable =
5850 C.getMBB()->isReturnBlock() && !Last->isCall()
5851 ? isLRAvailable(TRI, I: Last,
5852 E: (MachineBasicBlock::reverse_iterator)C.begin())
5853 : C.isAvailableAcrossAndOutOfSeq(Reg: ARM::LR, TRI);
5854 if (LRIsAvailable) {
5855 FrameID = MachineOutlinerNoLRSave;
5856 NumBytesNoStackCalls += Costs.CallNoLRSave;
5857 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: Costs.CallNoLRSave);
5858 CandidatesWithoutStackFixups.push_back(x: C);
5859 }
5860
5861 // Is an unused register available? If so, we won't modify the stack, so
5862 // we can outline with the same frame type as those that don't save LR.
5863 else if (findRegisterToSaveLRTo(C)) {
5864 FrameID = MachineOutlinerRegSave;
5865 NumBytesNoStackCalls += Costs.CallRegSave;
5866 C.setCallInfo(CID: MachineOutlinerRegSave, CO: Costs.CallRegSave);
5867 CandidatesWithoutStackFixups.push_back(x: C);
5868 }
5869
5870 // Is SP used in the sequence at all? If not, we don't have to modify
5871 // the stack, so we are guaranteed to get the same frame.
5872 else if (C.isAvailableInsideSeq(Reg: ARM::SP, TRI)) {
5873 NumBytesNoStackCalls += Costs.CallDefault;
5874 C.setCallInfo(CID: MachineOutlinerDefault, CO: Costs.CallDefault);
5875 CandidatesWithoutStackFixups.push_back(x: C);
5876 }
5877
5878 // If we outline this, we need to modify the stack. Pretend we don't
5879 // outline this by saving all of its bytes.
5880 else
5881 NumBytesNoStackCalls += SequenceSize;
5882 }
5883
5884 // If there are no places where we have to save LR, then note that we don't
5885 // have to update the stack. Otherwise, give every candidate the default
5886 // call type
5887 if (NumBytesNoStackCalls <=
5888 RepeatedSequenceLocs.size() * Costs.CallDefault) {
5889 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
5890 FrameID = MachineOutlinerNoLRSave;
5891 if (RepeatedSequenceLocs.size() < MinRepeats)
5892 return std::nullopt;
5893 } else
5894 SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault);
5895 }
5896
5897 // Does every candidate's MBB contain a call? If so, then we might have a
5898 // call in the range.
5899 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
5900 // check if the range contains a call. These require a save + restore of
5901 // the link register.
5902 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5903 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
5904 P: [](const MachineInstr &MI) { return MI.isCall(); }))
5905 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5906
5907 // Handle the last instruction separately. If it is tail call, then the
5908 // last instruction is a call, we don't want to save + restore in this
5909 // case. However, it could be possible that the last instruction is a
5910 // call without it being valid to tail call this sequence. We should
5911 // consider this as well.
5912 else if (FrameID != MachineOutlinerThunk &&
5913 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
5914 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5915 }
5916
5917 return std::make_unique<outliner::OutlinedFunction>(
5918 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
5919}
5920
5921bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI,
5922 int64_t Fixup,
5923 bool Updt) const {
5924 int SPIdx = MI->findRegisterUseOperandIdx(Reg: ARM::SP, /*TRI=*/nullptr);
5925 unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask);
5926 if (SPIdx < 0)
5927 // No SP operand
5928 return true;
5929 else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2))
5930 // If SP is not the base register we can't do much
5931 return false;
5932
5933 // Stack might be involved but addressing mode doesn't handle any offset.
5934 // Rq: AddrModeT1_[1|2|4] don't operate on SP
5935 if (AddrMode == ARMII::AddrMode1 || // Arithmetic instructions
5936 AddrMode == ARMII::AddrMode4 || // Load/Store Multiple
5937 AddrMode == ARMII::AddrMode6 || // Neon Load/Store Multiple
5938 AddrMode == ARMII::AddrModeT2_so || // SP can't be used as based register
5939 AddrMode == ARMII::AddrModeT2_pc || // PCrel access
5940 AddrMode == ARMII::AddrMode2 || // Used by PRE and POST indexed LD/ST
5941 AddrMode == ARMII::AddrModeT2_i7 || // v8.1-M MVE
5942 AddrMode == ARMII::AddrModeT2_i7s2 || // v8.1-M MVE
5943 AddrMode == ARMII::AddrModeT2_i7s4 || // v8.1-M sys regs VLDR/VSTR
5944 AddrMode == ARMII::AddrModeNone ||
5945 AddrMode == ARMII::AddrModeT2_i8 || // Pre/Post inc instructions
5946 AddrMode == ARMII::AddrModeT2_i8neg) // Always negative imm
5947 return false;
5948
5949 unsigned NumOps = MI->getDesc().getNumOperands();
5950 unsigned ImmIdx = NumOps - 3;
5951
5952 const MachineOperand &Offset = MI->getOperand(i: ImmIdx);
5953 assert(Offset.isImm() && "Is not an immediate");
5954 int64_t OffVal = Offset.getImm();
5955
5956 if (OffVal < 0)
5957 // Don't override data if the are below SP.
5958 return false;
5959
5960 unsigned NumBits = 0;
5961 unsigned Scale = 1;
5962
5963 switch (AddrMode) {
5964 case ARMII::AddrMode3:
5965 if (ARM_AM::getAM3Op(AM3Opc: OffVal) == ARM_AM::sub)
5966 return false;
5967 OffVal = ARM_AM::getAM3Offset(AM3Opc: OffVal);
5968 NumBits = 8;
5969 break;
5970 case ARMII::AddrMode5:
5971 if (ARM_AM::getAM5Op(AM5Opc: OffVal) == ARM_AM::sub)
5972 return false;
5973 OffVal = ARM_AM::getAM5Offset(AM5Opc: OffVal);
5974 NumBits = 8;
5975 Scale = 4;
5976 break;
5977 case ARMII::AddrMode5FP16:
5978 if (ARM_AM::getAM5FP16Op(AM5Opc: OffVal) == ARM_AM::sub)
5979 return false;
5980 OffVal = ARM_AM::getAM5FP16Offset(AM5Opc: OffVal);
5981 NumBits = 8;
5982 Scale = 2;
5983 break;
5984 case ARMII::AddrModeT2_i8pos:
5985 NumBits = 8;
5986 break;
5987 case ARMII::AddrModeT2_i8s4:
5988 // FIXME: Values are already scaled in this addressing mode.
5989 assert((Fixup & 3) == 0 && "Can't encode this offset!");
5990 NumBits = 10;
5991 break;
5992 case ARMII::AddrModeT2_ldrex:
5993 NumBits = 8;
5994 Scale = 4;
5995 break;
5996 case ARMII::AddrModeT2_i12:
5997 case ARMII::AddrMode_i12:
5998 NumBits = 12;
5999 break;
6000 case ARMII::AddrModeT1_s: // SP-relative LD/ST
6001 NumBits = 8;
6002 Scale = 4;
6003 break;
6004 default:
6005 llvm_unreachable("Unsupported addressing mode!");
6006 }
6007 // Make sure the offset is encodable for instructions that scale the
6008 // immediate.
6009 assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 &&
6010 "Can't encode this offset!");
6011 OffVal += Fixup / Scale;
6012
6013 unsigned Mask = (1 << NumBits) - 1;
6014
6015 if (OffVal <= Mask) {
6016 if (Updt)
6017 MI->getOperand(i: ImmIdx).setImm(OffVal);
6018 return true;
6019 }
6020
6021 return false;
6022}
6023
6024void ARMBaseInstrInfo::mergeOutliningCandidateAttributes(
6025 Function &F, std::vector<outliner::Candidate> &Candidates) const {
6026 outliner::Candidate &C = Candidates.front();
6027 // branch-target-enforcement is guaranteed to be consistent between all
6028 // candidates, so we only need to look at one.
6029 const Function &CFn = C.getMF()->getFunction();
6030 if (CFn.hasFnAttribute(Kind: "branch-target-enforcement"))
6031 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "branch-target-enforcement"));
6032
6033 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
6034 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
6035
6036 ARMGenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
6037}
6038
6039bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
6040 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6041 const Function &F = MF.getFunction();
6042
6043 // Can F be deduplicated by the linker? If it can, don't outline from it.
6044 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6045 return false;
6046
6047 // Don't outline from functions with section markings; the program could
6048 // expect that all the code is in the named section.
6049 // FIXME: Allow outlining from multiple functions with the same section
6050 // marking.
6051 if (F.hasSection())
6052 return false;
6053
6054 // FIXME: Thumb1 outlining is not handled
6055 if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
6056 return false;
6057
6058 // It's safe to outline from MF.
6059 return true;
6060}
6061
6062bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6063 unsigned &Flags) const {
6064 // Check if LR is available through all of the MBB. If it's not, then set
6065 // a flag.
6066 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6067 "Suitable Machine Function for outlining must track liveness");
6068
6069 LiveRegUnits LRU(getRegisterInfo());
6070
6071 for (MachineInstr &MI : llvm::reverse(C&: MBB))
6072 LRU.accumulate(MI);
6073
6074 // Check if each of the unsafe registers are available...
6075 bool R12AvailableInBlock = LRU.available(Reg: ARM::R12);
6076 bool CPSRAvailableInBlock = LRU.available(Reg: ARM::CPSR);
6077
6078 // If all of these are dead (and not live out), we know we don't have to check
6079 // them later.
6080 if (R12AvailableInBlock && CPSRAvailableInBlock)
6081 Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6082
6083 // Now, add the live outs to the set.
6084 LRU.addLiveOuts(MBB);
6085
6086 // If any of these registers is available in the MBB, but also a live out of
6087 // the block, then we know outlining is unsafe.
6088 if (R12AvailableInBlock && !LRU.available(Reg: ARM::R12))
6089 return false;
6090 if (CPSRAvailableInBlock && !LRU.available(Reg: ARM::CPSR))
6091 return false;
6092
6093 // Check if there's a call inside this MachineBasicBlock. If there is, then
6094 // set a flag.
6095 if (any_of(Range&: MBB, P: [](MachineInstr &MI) { return MI.isCall(); }))
6096 Flags |= MachineOutlinerMBBFlags::HasCalls;
6097
6098 // LR liveness is overestimated in return blocks.
6099
6100 bool LRIsAvailable =
6101 MBB.isReturnBlock() && !MBB.back().isCall()
6102 ? isLRAvailable(TRI: getRegisterInfo(), I: MBB.rbegin(), E: MBB.rend())
6103 : LRU.available(Reg: ARM::LR);
6104 if (!LRIsAvailable)
6105 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6106
6107 return true;
6108}
6109
6110outliner::InstrType
6111ARMBaseInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
6112 MachineBasicBlock::iterator &MIT,
6113 unsigned Flags) const {
6114 MachineInstr &MI = *MIT;
6115 const TargetRegisterInfo *TRI = &getRegisterInfo();
6116
6117 // PIC instructions contain labels, outlining them would break offset
6118 // computing. unsigned Opc = MI.getOpcode();
6119 unsigned Opc = MI.getOpcode();
6120 if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
6121 Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
6122 Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
6123 Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
6124 Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
6125 Opc == ARM::t2MOV_ga_pcrel)
6126 return outliner::InstrType::Illegal;
6127
6128 // Be conservative with ARMv8.1 MVE instructions.
6129 if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
6130 Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
6131 Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
6132 Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
6133 Opc == ARM::t2LoopEndDec)
6134 return outliner::InstrType::Illegal;
6135
6136 const MCInstrDesc &MCID = MI.getDesc();
6137 uint64_t MIFlags = MCID.TSFlags;
6138 if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
6139 return outliner::InstrType::Illegal;
6140
6141 // Is this a terminator for a basic block?
6142 if (MI.isTerminator())
6143 // TargetInstrInfo::getOutliningType has already filtered out anything
6144 // that would break this, so we can allow it here.
6145 return outliner::InstrType::Legal;
6146
6147 // Don't outline if link register or program counter value are used.
6148 if (MI.readsRegister(Reg: ARM::LR, TRI) || MI.readsRegister(Reg: ARM::PC, TRI))
6149 return outliner::InstrType::Illegal;
6150
6151 if (MI.isCall()) {
6152 // Get the function associated with the call. Look at each operand and find
6153 // the one that represents the calle and get its name.
6154 const Function *Callee = nullptr;
6155 for (const MachineOperand &MOP : MI.operands()) {
6156 if (MOP.isGlobal()) {
6157 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
6158 break;
6159 }
6160 }
6161
6162 // Dont't outline calls to "mcount" like functions, in particular Linux
6163 // kernel function tracing relies on it.
6164 if (Callee &&
6165 (Callee->getName() == "\01__gnu_mcount_nc" ||
6166 Callee->getName() == "\01mcount" || Callee->getName() == "__mcount"))
6167 return outliner::InstrType::Illegal;
6168
6169 // If we don't know anything about the callee, assume it depends on the
6170 // stack layout of the caller. In that case, it's only legal to outline
6171 // as a tail-call. Explicitly list the call instructions we know about so
6172 // we don't get unexpected results with call pseudo-instructions.
6173 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6174 if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX ||
6175 Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip ||
6176 Opc == ARM::tBLXi)
6177 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6178
6179 if (!Callee)
6180 return UnknownCallOutlineType;
6181
6182 // We have a function we have information about. Check if it's something we
6183 // can safely outline.
6184 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
6185
6186 // We don't know what's going on with the callee at all. Don't touch it.
6187 if (!CalleeMF)
6188 return UnknownCallOutlineType;
6189
6190 // Check if we know anything about the callee saves on the function. If we
6191 // don't, then don't touch it, since that implies that we haven't computed
6192 // anything about its stack frame yet.
6193 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6194 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6195 MFI.getNumObjects() > 0)
6196 return UnknownCallOutlineType;
6197
6198 // At this point, we can say that CalleeMF ought to not pass anything on the
6199 // stack. Therefore, we can outline it.
6200 return outliner::InstrType::Legal;
6201 }
6202
6203 // Since calls are handled, don't touch LR or PC
6204 if (MI.modifiesRegister(Reg: ARM::LR, TRI) || MI.modifiesRegister(Reg: ARM::PC, TRI))
6205 return outliner::InstrType::Illegal;
6206
6207 // Does this use the stack?
6208 if (MI.modifiesRegister(Reg: ARM::SP, TRI) || MI.readsRegister(Reg: ARM::SP, TRI)) {
6209 // True if there is no chance that any outlined candidate from this range
6210 // could require stack fixups. That is, both
6211 // * LR is available in the range (No save/restore around call)
6212 // * The range doesn't include calls (No save/restore in outlined frame)
6213 // are true.
6214 // These conditions also ensure correctness of the return address
6215 // authentication - we insert sign and authentication instructions only if
6216 // we save/restore LR on stack, but then this condition ensures that the
6217 // outlined range does not modify the SP, therefore the SP value used for
6218 // signing is the same as the one used for authentication.
6219 // FIXME: This is very restrictive; the flags check the whole block,
6220 // not just the bit we will try to outline.
6221 bool MightNeedStackFixUp =
6222 (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere |
6223 MachineOutlinerMBBFlags::HasCalls));
6224
6225 if (!MightNeedStackFixUp)
6226 return outliner::InstrType::Legal;
6227
6228 // Any modification of SP will break our code to save/restore LR.
6229 // FIXME: We could handle some instructions which add a constant offset to
6230 // SP, with a bit more work.
6231 if (MI.modifiesRegister(Reg: ARM::SP, TRI))
6232 return outliner::InstrType::Illegal;
6233
6234 // At this point, we have a stack instruction that we might need to fix up.
6235 // up. We'll handle it if it's a load or store.
6236 if (checkAndUpdateStackOffset(MI: &MI, Fixup: Subtarget.getStackAlignment().value(),
6237 Updt: false))
6238 return outliner::InstrType::Legal;
6239
6240 // We can't fix it up, so don't outline it.
6241 return outliner::InstrType::Illegal;
6242 }
6243
6244 // Be conservative with IT blocks.
6245 if (MI.readsRegister(Reg: ARM::ITSTATE, TRI) ||
6246 MI.modifiesRegister(Reg: ARM::ITSTATE, TRI))
6247 return outliner::InstrType::Illegal;
6248
6249 // Don't outline CFI instructions.
6250 if (MI.isCFIInstruction())
6251 return outliner::InstrType::Illegal;
6252
6253 return outliner::InstrType::Legal;
6254}
6255
6256void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6257 for (MachineInstr &MI : MBB) {
6258 checkAndUpdateStackOffset(MI: &MI, Fixup: Subtarget.getStackAlignment().value(), Updt: true);
6259 }
6260}
6261
6262void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
6263 MachineBasicBlock::iterator It, bool CFI,
6264 bool Auth) const {
6265 int Align = std::max(a: Subtarget.getStackAlignment().value(), b: uint64_t(8));
6266 unsigned MIFlags = CFI ? MachineInstr::FrameSetup : 0;
6267 assert(Align >= 8 && Align <= 256);
6268 if (Auth) {
6269 assert(Subtarget.isThumb2());
6270 // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
6271 // sequence.
6272 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2PAC)).setMIFlags(MIFlags);
6273 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2STRD_PRE), DestReg: ARM::SP)
6274 .addReg(RegNo: ARM::R12, Flags: RegState::Kill)
6275 .addReg(RegNo: ARM::LR, Flags: RegState::Kill)
6276 .addReg(RegNo: ARM::SP)
6277 .addImm(Val: -Align)
6278 .add(MOs: predOps(Pred: ARMCC::AL))
6279 .setMIFlags(MIFlags);
6280 } else {
6281 unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
6282 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg: ARM::SP)
6283 .addReg(RegNo: ARM::LR, Flags: RegState::Kill)
6284 .addReg(RegNo: ARM::SP)
6285 .addImm(Val: -Align)
6286 .add(MOs: predOps(Pred: ARMCC::AL))
6287 .setMIFlags(MIFlags);
6288 }
6289
6290 if (!CFI)
6291 return;
6292
6293 // Add a CFI, saying CFA is offset by Align bytes from SP.
6294 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
6295 CFIBuilder.buildDefCFAOffset(Offset: Align);
6296
6297 // Add a CFI saying that the LR that we want to find is now higher than
6298 // before.
6299 int LROffset = Auth ? Align - 4 : Align;
6300 CFIBuilder.buildOffset(Reg: ARM::LR, Offset: -LROffset);
6301 if (Auth) {
6302 // Add a CFI for the location of the return address PAC.
6303 CFIBuilder.buildOffset(Reg: ARM::RA_AUTH_CODE, Offset: -Align);
6304 }
6305}
6306
6307void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
6308 MachineBasicBlock::iterator It,
6309 bool CFI, bool Auth) const {
6310 int Align = Subtarget.getStackAlignment().value();
6311 unsigned MIFlags = CFI ? MachineInstr::FrameDestroy : 0;
6312 if (Auth) {
6313 assert(Subtarget.isThumb2());
6314 // Restore return address PAC and LR.
6315 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2LDRD_POST))
6316 .addReg(RegNo: ARM::R12, Flags: RegState::Define)
6317 .addReg(RegNo: ARM::LR, Flags: RegState::Define)
6318 .addReg(RegNo: ARM::SP, Flags: RegState::Define)
6319 .addReg(RegNo: ARM::SP)
6320 .addImm(Val: Align)
6321 .add(MOs: predOps(Pred: ARMCC::AL))
6322 .setMIFlags(MIFlags);
6323 // LR authentication is after the CFI instructions, below.
6324 } else {
6325 unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
6326 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg: ARM::LR)
6327 .addReg(RegNo: ARM::SP, Flags: RegState::Define)
6328 .addReg(RegNo: ARM::SP);
6329 if (!Subtarget.isThumb())
6330 MIB.addReg(RegNo: 0);
6331 MIB.addImm(Val: Subtarget.getStackAlignment().value())
6332 .add(MOs: predOps(Pred: ARMCC::AL))
6333 .setMIFlags(MIFlags);
6334 }
6335
6336 if (CFI) {
6337 // Now stack has moved back up and we have restored LR.
6338 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameDestroy);
6339 CFIBuilder.buildDefCFAOffset(Offset: 0);
6340 CFIBuilder.buildRestore(Reg: ARM::LR);
6341 if (Auth)
6342 CFIBuilder.buildUndefined(Reg: ARM::RA_AUTH_CODE);
6343 }
6344
6345 if (Auth)
6346 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2AUT));
6347}
6348
6349void ARMBaseInstrInfo::buildOutlinedFrame(
6350 MachineBasicBlock &MBB, MachineFunction &MF,
6351 const outliner::OutlinedFunction &OF) const {
6352 // For thunk outlining, rewrite the last instruction from a call to a
6353 // tail-call.
6354 if (OF.FrameConstructionID == MachineOutlinerThunk) {
6355 MachineInstr *Call = &*--MBB.instr_end();
6356 bool isThumb = Subtarget.isThumb();
6357 unsigned FuncOp = isThumb ? 2 : 0;
6358 unsigned Opc = Call->getOperand(i: FuncOp).isReg()
6359 ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
6360 : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
6361 : ARM::tTAILJMPdND
6362 : ARM::TAILJMPd;
6363 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6364 .add(MO: Call->getOperand(i: FuncOp));
6365 if (isThumb && !Call->getOperand(i: FuncOp).isReg())
6366 MIB.add(MOs: predOps(Pred: ARMCC::AL));
6367 Call->eraseFromParent();
6368 }
6369
6370 // Is there a call in the outlined range?
6371 auto IsNonTailCall = [](MachineInstr &MI) {
6372 return MI.isCall() && !MI.isReturn();
6373 };
6374 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
6375 MachineBasicBlock::iterator It = MBB.begin();
6376 MachineBasicBlock::iterator Et = MBB.end();
6377
6378 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6379 OF.FrameConstructionID == MachineOutlinerThunk)
6380 Et = std::prev(x: MBB.end());
6381
6382 // We have to save and restore LR, we need to add it to the liveins if it
6383 // is not already part of the set. This is sufficient since outlined
6384 // functions only have one block.
6385 if (!MBB.isLiveIn(Reg: ARM::LR))
6386 MBB.addLiveIn(PhysReg: ARM::LR);
6387
6388 // Insert a save before the outlined region
6389 bool Auth = MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(SpillsLR: true);
6390 saveLROnStack(MBB, It, CFI: true, Auth);
6391
6392 // Fix up the instructions in the range, since we're going to modify the
6393 // stack.
6394 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6395 "Can only fix up stack references once");
6396 fixupPostOutline(MBB);
6397
6398 // Insert a restore before the terminator for the function. Restore LR.
6399 restoreLRFromStack(MBB, It: Et, CFI: true, Auth);
6400 }
6401
6402 // If this is a tail call outlined function, then there's already a return.
6403 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6404 OF.FrameConstructionID == MachineOutlinerThunk)
6405 return;
6406
6407 // Here we have to insert the return ourselves. Get the correct opcode from
6408 // current feature set.
6409 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: Subtarget.getReturnOpcode()))
6410 .add(MOs: predOps(Pred: ARMCC::AL));
6411
6412 // Did we have to modify the stack by saving the link register?
6413 if (OF.FrameConstructionID != MachineOutlinerDefault &&
6414 OF.Candidates[0].CallConstructionID != MachineOutlinerDefault)
6415 return;
6416
6417 // We modified the stack.
6418 // Walk over the basic block and fix up all the stack accesses.
6419 fixupPostOutline(MBB);
6420}
6421
6422MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
6423 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6424 MachineFunction &MF, outliner::Candidate &C) const {
6425 MachineInstrBuilder MIB;
6426 MachineBasicBlock::iterator CallPt;
6427 unsigned Opc;
6428 bool isThumb = Subtarget.isThumb();
6429
6430 // Are we tail calling?
6431 if (C.CallConstructionID == MachineOutlinerTailCall) {
6432 // If yes, then we can just branch to the label.
6433 Opc = isThumb
6434 ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
6435 : ARM::TAILJMPd;
6436 MIB = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6437 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()));
6438 if (isThumb)
6439 MIB.add(MOs: predOps(Pred: ARMCC::AL));
6440 It = MBB.insert(I: It, MI: MIB);
6441 return It;
6442 }
6443
6444 // Create the call instruction.
6445 Opc = isThumb ? ARM::tBL : ARM::BL;
6446 MachineInstrBuilder CallMIB = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: Opc));
6447 if (isThumb)
6448 CallMIB.add(MOs: predOps(Pred: ARMCC::AL));
6449 CallMIB.addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()));
6450
6451 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6452 C.CallConstructionID == MachineOutlinerThunk) {
6453 // No, so just insert the call.
6454 It = MBB.insert(I: It, MI: CallMIB);
6455 return It;
6456 }
6457
6458 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
6459 // Can we save to a register?
6460 if (C.CallConstructionID == MachineOutlinerRegSave) {
6461 Register Reg = findRegisterToSaveLRTo(C);
6462 assert(Reg != 0 && "No callee-saved register available?");
6463
6464 // Save and restore LR from that register.
6465 copyPhysReg(MBB, I: It, DL: DebugLoc(), DestReg: Reg, SrcReg: ARM::LR, KillSrc: true);
6466 if (!AFI.isLRSpilled())
6467 CFIInstBuilder(MBB, It, MachineInstr::FrameSetup)
6468 .buildRegister(Reg1: ARM::LR, Reg2: Reg);
6469 CallPt = MBB.insert(I: It, MI: CallMIB);
6470 copyPhysReg(MBB, I: It, DL: DebugLoc(), DestReg: ARM::LR, SrcReg: Reg, KillSrc: true);
6471 if (!AFI.isLRSpilled())
6472 CFIInstBuilder(MBB, It, MachineInstr::FrameDestroy).buildRestore(Reg: ARM::LR);
6473 It--;
6474 return CallPt;
6475 }
6476 // We have the default case. Save and restore from SP.
6477 if (!MBB.isLiveIn(Reg: ARM::LR))
6478 MBB.addLiveIn(PhysReg: ARM::LR);
6479 bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(SpillsLR: true);
6480 saveLROnStack(MBB, It, CFI: !AFI.isLRSpilled(), Auth);
6481 CallPt = MBB.insert(I: It, MI: CallMIB);
6482 restoreLRFromStack(MBB, It, CFI: !AFI.isLRSpilled(), Auth);
6483 It--;
6484 return CallPt;
6485}
6486
6487bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault(
6488 MachineFunction &MF) const {
6489 return Subtarget.isMClass() && MF.getFunction().hasMinSize();
6490}
6491
6492bool ARMBaseInstrInfo::isReMaterializableImpl(
6493 const MachineInstr &MI) const {
6494 // Try hard to rematerialize any VCTPs because if we spill P0, it will block
6495 // the tail predication conversion. This means that the element count
6496 // register has to be live for longer, but that has to be better than
6497 // spill/restore and VPT predication.
6498 return (isVCTP(MI: &MI) && !isPredicated(MI)) ||
6499 TargetInstrInfo::isReMaterializableImpl(MI);
6500}
6501
6502unsigned llvm::getBLXOpcode(const MachineFunction &MF) {
6503 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip
6504 : ARM::BLX;
6505}
6506
6507unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) {
6508 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip
6509 : ARM::tBLXr;
6510}
6511
6512unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) {
6513 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip
6514 : ARM::BLX_pred;
6515}
6516
6517namespace {
6518class ARMPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
6519 MachineInstr *EndLoop, *LoopCount;
6520 MachineFunction *MF;
6521 const TargetInstrInfo *TII;
6522
6523 // Bitset[0 .. MAX_STAGES-1] ... iterations needed
6524 // [LAST_IS_USE] : last reference to register in schedule is a use
6525 // [SEEN_AS_LIVE] : Normal pressure algorithm believes register is live
6526 static int constexpr MAX_STAGES = 30;
6527 static int constexpr LAST_IS_USE = MAX_STAGES;
6528 static int constexpr SEEN_AS_LIVE = MAX_STAGES + 1;
6529 typedef std::bitset<MAX_STAGES + 2> IterNeed;
6530 typedef std::map<Register, IterNeed> IterNeeds;
6531
6532 void bumpCrossIterationPressure(RegPressureTracker &RPT,
6533 const IterNeeds &CIN);
6534 bool tooMuchRegisterPressure(SwingSchedulerDAG &SSD, SMSchedule &SMS);
6535
6536 // Meanings of the various stuff with loop types:
6537 // t2Bcc:
6538 // EndLoop = branch at end of original BB that will become a kernel
6539 // LoopCount = CC setter live into branch
6540 // t2LoopEnd:
6541 // EndLoop = branch at end of original BB
6542 // LoopCount = t2LoopDec
6543public:
6544 ARMPipelinerLoopInfo(MachineInstr *EndLoop, MachineInstr *LoopCount)
6545 : EndLoop(EndLoop), LoopCount(LoopCount),
6546 MF(EndLoop->getParent()->getParent()),
6547 TII(MF->getSubtarget().getInstrInfo()) {}
6548
6549 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
6550 // Only ignore the terminator.
6551 return MI == EndLoop || MI == LoopCount;
6552 }
6553
6554 bool shouldUseSchedule(SwingSchedulerDAG &SSD, SMSchedule &SMS) override {
6555 if (tooMuchRegisterPressure(SSD, SMS))
6556 return false;
6557
6558 return true;
6559 }
6560
6561 std::optional<bool> createTripCountGreaterCondition(
6562 int TC, MachineBasicBlock &MBB,
6563 SmallVectorImpl<MachineOperand> &Cond) override {
6564
6565 if (isCondBranchOpcode(Opc: EndLoop->getOpcode())) {
6566 Cond.push_back(Elt: EndLoop->getOperand(i: 1));
6567 Cond.push_back(Elt: EndLoop->getOperand(i: 2));
6568 if (EndLoop->getOperand(i: 0).getMBB() == EndLoop->getParent()) {
6569 TII->reverseBranchCondition(Cond);
6570 }
6571 return {};
6572 } else if (EndLoop->getOpcode() == ARM::t2LoopEnd) {
6573 // General case just lets the unrolled t2LoopDec do the subtraction and
6574 // therefore just needs to check if zero has been reached.
6575 MachineInstr *LoopDec = nullptr;
6576 for (auto &I : MBB.instrs())
6577 if (I.getOpcode() == ARM::t2LoopDec)
6578 LoopDec = &I;
6579 assert(LoopDec && "Unable to find copied LoopDec");
6580 // Check if we're done with the loop.
6581 BuildMI(BB: &MBB, MIMD: LoopDec->getDebugLoc(), MCID: TII->get(Opcode: ARM::t2CMPri))
6582 .addReg(RegNo: LoopDec->getOperand(i: 0).getReg())
6583 .addImm(Val: 0)
6584 .addImm(Val: ARMCC::AL)
6585 .addReg(RegNo: ARM::NoRegister);
6586 Cond.push_back(Elt: MachineOperand::CreateImm(Val: ARMCC::EQ));
6587 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: ARM::CPSR, isDef: false));
6588 return {};
6589 } else
6590 llvm_unreachable("Unknown EndLoop");
6591 }
6592
6593 void setPreheader(MachineBasicBlock *NewPreheader) override {}
6594
6595 void adjustTripCount(int TripCountAdjust) override {}
6596};
6597
6598void ARMPipelinerLoopInfo::bumpCrossIterationPressure(RegPressureTracker &RPT,
6599 const IterNeeds &CIN) {
6600 // Increase pressure by the amounts in CrossIterationNeeds
6601 for (const auto &N : CIN) {
6602 int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6603 for (int I = 0; I < Cnt; ++I)
6604 RPT.increaseRegPressure(VRegOrUnit: VirtRegOrUnit(N.first), PreviousMask: LaneBitmask::getNone(),
6605 NewMask: LaneBitmask::getAll());
6606 }
6607 // Decrease pressure by the amounts in CrossIterationNeeds
6608 for (const auto &N : CIN) {
6609 int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6610 for (int I = 0; I < Cnt; ++I)
6611 RPT.decreaseRegPressure(VRegOrUnit: VirtRegOrUnit(N.first), PreviousMask: LaneBitmask::getAll(),
6612 NewMask: LaneBitmask::getNone());
6613 }
6614}
6615
6616bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
6617 SMSchedule &SMS) {
6618 IterNeeds CrossIterationNeeds;
6619
6620 // Determine which values will be loop-carried after the schedule is
6621 // applied
6622
6623 for (auto &SU : SSD.SUnits) {
6624 const MachineInstr *MI = SU.getInstr();
6625 int Stg = SMS.stageScheduled(SU: const_cast<SUnit *>(&SU));
6626 for (auto &S : SU.Succs)
6627 if (MI->isPHI() && S.getKind() == SDep::Anti) {
6628 Register Reg = S.getReg();
6629 if (Reg.isVirtual())
6630 CrossIterationNeeds[Reg.id()].set(position: 0);
6631 } else if (S.isAssignedRegDep()) {
6632 int OStg = SMS.stageScheduled(SU: S.getSUnit());
6633 if (OStg >= 0 && OStg != Stg) {
6634 Register Reg = S.getReg();
6635 if (Reg.isVirtual())
6636 CrossIterationNeeds[Reg.id()] |= ((1 << (OStg - Stg)) - 1);
6637 }
6638 }
6639 }
6640
6641 // Determine more-or-less what the proposed schedule (reversed) is going to
6642 // be; it might not be quite the same because the within-cycle ordering
6643 // created by SMSchedule depends upon changes to help with address offsets and
6644 // the like.
6645 std::vector<SUnit *> ProposedSchedule;
6646 for (int Cycle = SMS.getFinalCycle(); Cycle >= SMS.getFirstCycle(); --Cycle)
6647 for (int Stage = 0, StageEnd = SMS.getMaxStageCount(); Stage <= StageEnd;
6648 ++Stage) {
6649 std::deque<SUnit *> Instrs =
6650 SMS.getInstructions(cycle: Cycle + Stage * SMS.getInitiationInterval());
6651 std::sort(first: Instrs.begin(), last: Instrs.end(),
6652 comp: [](SUnit *A, SUnit *B) { return A->NodeNum > B->NodeNum; });
6653 llvm::append_range(C&: ProposedSchedule, R&: Instrs);
6654 }
6655
6656 // Learn whether the last use/def of each cross-iteration register is a use or
6657 // def. If it is a def, RegisterPressure will implicitly increase max pressure
6658 // and we do not have to add the pressure.
6659 for (auto *SU : ProposedSchedule)
6660 for (ConstMIBundleOperands OperI(*SU->getInstr()); OperI.isValid();
6661 ++OperI) {
6662 auto MO = *OperI;
6663 if (!MO.isReg() || !MO.getReg())
6664 continue;
6665 Register Reg = MO.getReg();
6666 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6667 if (CIter == CrossIterationNeeds.end() || CIter->second[LAST_IS_USE] ||
6668 CIter->second[SEEN_AS_LIVE])
6669 continue;
6670 if (MO.isDef() && !MO.isDead())
6671 CIter->second.set(position: SEEN_AS_LIVE);
6672 else if (MO.isUse())
6673 CIter->second.set(position: LAST_IS_USE);
6674 }
6675 for (auto &CI : CrossIterationNeeds)
6676 CI.second.reset(position: LAST_IS_USE);
6677
6678 RegionPressure RecRegPressure;
6679 RegPressureTracker RPTracker(RecRegPressure);
6680 RegisterClassInfo RegClassInfo;
6681 RegClassInfo.runOnMachineFunction(MF: *MF);
6682 RPTracker.init(mf: MF, rci: &RegClassInfo, lis: nullptr, mbb: EndLoop->getParent(),
6683 pos: EndLoop->getParent()->end(), TrackLaneMasks: false, TrackUntiedDefs: false);
6684
6685 bumpCrossIterationPressure(RPT&: RPTracker, CIN: CrossIterationNeeds);
6686
6687 for (auto *SU : ProposedSchedule) {
6688 MachineBasicBlock::const_iterator CurInstI = SU->getInstr();
6689 RPTracker.setPos(std::next(x: CurInstI));
6690 RPTracker.recede();
6691
6692 // Track what cross-iteration registers would be seen as live
6693 for (ConstMIBundleOperands OperI(*CurInstI); OperI.isValid(); ++OperI) {
6694 auto MO = *OperI;
6695 if (!MO.isReg() || !MO.getReg())
6696 continue;
6697 Register Reg = MO.getReg();
6698 if (MO.isDef() && !MO.isDead()) {
6699 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6700 if (CIter != CrossIterationNeeds.end()) {
6701 CIter->second.reset(position: 0);
6702 CIter->second.reset(position: SEEN_AS_LIVE);
6703 }
6704 }
6705 }
6706 for (auto &S : SU->Preds) {
6707 auto Stg = SMS.stageScheduled(SU);
6708 if (S.isAssignedRegDep()) {
6709 Register Reg = S.getReg();
6710 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6711 if (CIter != CrossIterationNeeds.end()) {
6712 auto Stg2 = SMS.stageScheduled(SU: S.getSUnit());
6713 assert(Stg2 <= Stg && "Data dependence upon earlier stage");
6714 if (Stg - Stg2 < MAX_STAGES)
6715 CIter->second.set(position: Stg - Stg2);
6716 CIter->second.set(position: SEEN_AS_LIVE);
6717 }
6718 }
6719 }
6720
6721 bumpCrossIterationPressure(RPT&: RPTracker, CIN: CrossIterationNeeds);
6722 }
6723
6724 auto &P = RPTracker.getPressure().MaxSetPressure;
6725 for (unsigned I = 0, E = P.size(); I < E; ++I) {
6726 // Exclude some Neon register classes.
6727 if (I == ARM::DQuad_with_ssub_0 || I == ARM::DTripleSpc_with_ssub_0 ||
6728 I == ARM::DTriple_with_qsub_0_in_QPR)
6729 continue;
6730
6731 if (P[I] > RegClassInfo.getRegPressureSetLimit(Idx: I)) {
6732 return true;
6733 }
6734 }
6735 return false;
6736}
6737
6738} // namespace
6739
6740std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
6741ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
6742 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
6743 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
6744 if (Preheader == LoopBB)
6745 Preheader = *std::next(x: LoopBB->pred_begin());
6746
6747 if (I != LoopBB->end() && I->getOpcode() == ARM::t2Bcc) {
6748 // If the branch is a Bcc, then the CPSR should be set somewhere within the
6749 // block. We need to determine the reaching definition of CPSR so that
6750 // it can be marked as non-pipelineable, allowing the pipeliner to force
6751 // it into stage 0 or give up if it cannot or will not do so.
6752 MachineInstr *CCSetter = nullptr;
6753 for (auto &L : LoopBB->instrs()) {
6754 if (L.isCall())
6755 return nullptr;
6756 if (isCPSRDefined(MI: L))
6757 CCSetter = &L;
6758 }
6759 if (CCSetter)
6760 return std::make_unique<ARMPipelinerLoopInfo>(args: &*I, args&: CCSetter);
6761 else
6762 return nullptr; // Unable to find the CC setter, so unable to guarantee
6763 // that pipeline will work
6764 }
6765
6766 // Recognize:
6767 // preheader:
6768 // %1 = t2DoopLoopStart %0
6769 // loop:
6770 // %2 = phi %1, <not loop>, %..., %loop
6771 // %3 = t2LoopDec %2, <imm>
6772 // t2LoopEnd %3, %loop
6773
6774 if (I != LoopBB->end() && I->getOpcode() == ARM::t2LoopEnd) {
6775 for (auto &L : LoopBB->instrs())
6776 if (L.isCall())
6777 return nullptr;
6778 else if (isVCTP(MI: &L))
6779 return nullptr;
6780 Register LoopDecResult = I->getOperand(i: 0).getReg();
6781 MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
6782 MachineInstr *LoopDec = MRI.getUniqueVRegDef(Reg: LoopDecResult);
6783 if (!LoopDec || LoopDec->getOpcode() != ARM::t2LoopDec)
6784 return nullptr;
6785 MachineInstr *LoopStart = nullptr;
6786 for (auto &J : Preheader->instrs())
6787 if (J.getOpcode() == ARM::t2DoLoopStart)
6788 LoopStart = &J;
6789 if (!LoopStart)
6790 return nullptr;
6791 return std::make_unique<ARMPipelinerLoopInfo>(args: &*I, args&: LoopDec);
6792 }
6793 return nullptr;
6794}
6795