1//===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the Base ARM implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ARMBaseInstrInfo.h"
14#include "ARMBaseRegisterInfo.h"
15#include "ARMConstantPoolValue.h"
16#include "ARMFeatures.h"
17#include "ARMHazardRecognizer.h"
18#include "ARMMachineFunctionInfo.h"
19#include "ARMSubtarget.h"
20#include "MCTargetDesc/ARMAddressingModes.h"
21#include "MCTargetDesc/ARMBaseInfo.h"
22#include "MVETailPredUtils.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/DFAPacketizer.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineConstantPool.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineMemOperand.h"
36#include "llvm/CodeGen/MachineModuleInfo.h"
37#include "llvm/CodeGen/MachineOperand.h"
38#include "llvm/CodeGen/MachinePipeliner.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/MachineScheduler.h"
41#include "llvm/CodeGen/MultiHazardRecognizer.h"
42#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/TargetInstrInfo.h"
45#include "llvm/CodeGen/TargetRegisterInfo.h"
46#include "llvm/CodeGen/TargetSchedule.h"
47#include "llvm/IR/Attributes.h"
48#include "llvm/IR/DebugLoc.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/Module.h"
52#include "llvm/MC/MCAsmInfo.h"
53#include "llvm/MC/MCInstrDesc.h"
54#include "llvm/MC/MCInstrItineraries.h"
55#include "llvm/Support/BranchProbability.h"
56#include "llvm/Support/Casting.h"
57#include "llvm/Support/Compiler.h"
58#include "llvm/Support/Debug.h"
59#include "llvm/Support/ErrorHandling.h"
60#include "llvm/Support/raw_ostream.h"
61#include "llvm/Target/TargetMachine.h"
62#include <algorithm>
63#include <cassert>
64#include <cstdint>
65#include <iterator>
66#include <new>
67#include <utility>
68#include <vector>
69
70using namespace llvm;
71
72#define DEBUG_TYPE "arm-instrinfo"
73
74#define GET_INSTRINFO_CTOR_DTOR
75#include "ARMGenInstrInfo.inc"
76
77/// ARM_MLxEntry - Record information about MLA / MLS instructions.
78struct ARM_MLxEntry {
79 uint16_t MLxOpc; // MLA / MLS opcode
80 uint16_t MulOpc; // Expanded multiplication opcode
81 uint16_t AddSubOpc; // Expanded add / sub opcode
82 bool NegAcc; // True if the acc is negated before the add / sub.
83 bool HasLane; // True if instruction has an extra "lane" operand.
84};
85
86static const ARM_MLxEntry ARM_MLxTable[] = {
87 // MLxOpc, MulOpc, AddSubOpc, NegAcc, HasLane
88 // fp scalar ops
89 { .MLxOpc: ARM::VMLAS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VADDS, .NegAcc: false, .HasLane: false },
90 { .MLxOpc: ARM::VMLSS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: false, .HasLane: false },
91 { .MLxOpc: ARM::VMLAD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VADDD, .NegAcc: false, .HasLane: false },
92 { .MLxOpc: ARM::VMLSD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: false, .HasLane: false },
93 { .MLxOpc: ARM::VNMLAS, .MulOpc: ARM::VNMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: true, .HasLane: false },
94 { .MLxOpc: ARM::VNMLSS, .MulOpc: ARM::VMULS, .AddSubOpc: ARM::VSUBS, .NegAcc: true, .HasLane: false },
95 { .MLxOpc: ARM::VNMLAD, .MulOpc: ARM::VNMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: true, .HasLane: false },
96 { .MLxOpc: ARM::VNMLSD, .MulOpc: ARM::VMULD, .AddSubOpc: ARM::VSUBD, .NegAcc: true, .HasLane: false },
97
98 // fp SIMD ops
99 { .MLxOpc: ARM::VMLAfd, .MulOpc: ARM::VMULfd, .AddSubOpc: ARM::VADDfd, .NegAcc: false, .HasLane: false },
100 { .MLxOpc: ARM::VMLSfd, .MulOpc: ARM::VMULfd, .AddSubOpc: ARM::VSUBfd, .NegAcc: false, .HasLane: false },
101 { .MLxOpc: ARM::VMLAfq, .MulOpc: ARM::VMULfq, .AddSubOpc: ARM::VADDfq, .NegAcc: false, .HasLane: false },
102 { .MLxOpc: ARM::VMLSfq, .MulOpc: ARM::VMULfq, .AddSubOpc: ARM::VSUBfq, .NegAcc: false, .HasLane: false },
103 { .MLxOpc: ARM::VMLAslfd, .MulOpc: ARM::VMULslfd, .AddSubOpc: ARM::VADDfd, .NegAcc: false, .HasLane: true },
104 { .MLxOpc: ARM::VMLSslfd, .MulOpc: ARM::VMULslfd, .AddSubOpc: ARM::VSUBfd, .NegAcc: false, .HasLane: true },
105 { .MLxOpc: ARM::VMLAslfq, .MulOpc: ARM::VMULslfq, .AddSubOpc: ARM::VADDfq, .NegAcc: false, .HasLane: true },
106 { .MLxOpc: ARM::VMLSslfq, .MulOpc: ARM::VMULslfq, .AddSubOpc: ARM::VSUBfq, .NegAcc: false, .HasLane: true },
107};
108
109ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget &STI,
110 const ARMBaseRegisterInfo &TRI)
111 : ARMGenInstrInfo(STI, TRI, ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
112 Subtarget(STI) {
113 for (unsigned i = 0, e = std::size(ARM_MLxTable); i != e; ++i) {
114 if (!MLxEntryMap.insert(KV: std::make_pair(x: ARM_MLxTable[i].MLxOpc, y&: i)).second)
115 llvm_unreachable("Duplicated entries?");
116 MLxHazardOpcodes.insert(V: ARM_MLxTable[i].AddSubOpc);
117 MLxHazardOpcodes.insert(V: ARM_MLxTable[i].MulOpc);
118 }
119}
120
121// Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
122// currently defaults to no prepass hazard recognizer.
123ScheduleHazardRecognizer *
124ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
125 const ScheduleDAG *DAG) const {
126 if (usePreRAHazardRecognizer()) {
127 const InstrItineraryData *II =
128 static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
129 return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
130 }
131 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
132}
133
134// Called during:
135// - pre-RA scheduling
136// - post-RA scheduling when FeatureUseMISched is set
137ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer(
138 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
139 MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
140
141 // We would like to restrict this hazard recognizer to only
142 // post-RA scheduling; we can tell that we're post-RA because we don't
143 // track VRegLiveness.
144 // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM
145 // banks banked on bit 2. Assume that TCMs are in use.
146 if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness())
147 MHR->AddHazardRecognizer(
148 std::make_unique<ARMBankConflictHazardRecognizer>(args&: DAG, args: 0x4, args: true));
149
150 // Not inserting ARMHazardRecognizerFPMLx because that would change
151 // legacy behavior
152
153 auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
154 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
155 return MHR;
156}
157
158// Called during post-RA scheduling when FeatureUseMISched is not set
159ScheduleHazardRecognizer *ARMBaseInstrInfo::
160CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
161 const ScheduleDAG *DAG) const {
162 MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
163
164 if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
165 MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>());
166
167 auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
168 if (BHR)
169 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
170 return MHR;
171}
172
173// Branch analysis.
174// Cond vector output format:
175// 0 elements indicates an unconditional branch
176// 2 elements indicates a conditional branch; the elements are
177// the condition to check and the CPSR.
178// 3 elements indicates a hardware loop end; the elements
179// are the opcode, the operand value to test, and a dummy
180// operand used to pad out to 3 operands.
181bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
182 MachineBasicBlock *&TBB,
183 MachineBasicBlock *&FBB,
184 SmallVectorImpl<MachineOperand> &Cond,
185 bool AllowModify) const {
186 TBB = nullptr;
187 FBB = nullptr;
188
189 MachineBasicBlock::instr_iterator I = MBB.instr_end();
190 if (I == MBB.instr_begin())
191 return false; // Empty blocks are easy.
192 --I;
193
194 // Walk backwards from the end of the basic block until the branch is
195 // analyzed or we give up.
196 while (isPredicated(MI: *I) || I->isTerminator() || I->isDebugValue()) {
197 // Flag to be raised on unanalyzeable instructions. This is useful in cases
198 // where we want to clean up on the end of the basic block before we bail
199 // out.
200 bool CantAnalyze = false;
201
202 // Skip over DEBUG values, predicated nonterminators and speculation
203 // barrier terminators.
204 while (I->isDebugInstr() || !I->isTerminator() ||
205 isSpeculationBarrierEndBBOpcode(Opc: I->getOpcode()) ||
206 I->getOpcode() == ARM::t2DoLoopStartTP){
207 if (I == MBB.instr_begin())
208 return false;
209 --I;
210 }
211
212 if (isIndirectBranchOpcode(Opc: I->getOpcode()) ||
213 isJumpTableBranchOpcode(Opc: I->getOpcode())) {
214 // Indirect branches and jump tables can't be analyzed, but we still want
215 // to clean up any instructions at the tail of the basic block.
216 CantAnalyze = true;
217 } else if (isUncondBranchOpcode(Opc: I->getOpcode())) {
218 TBB = I->getOperand(i: 0).getMBB();
219 } else if (isCondBranchOpcode(Opc: I->getOpcode())) {
220 // Bail out if we encounter multiple conditional branches.
221 if (!Cond.empty())
222 return true;
223
224 assert(!FBB && "FBB should have been null.");
225 FBB = TBB;
226 TBB = I->getOperand(i: 0).getMBB();
227 Cond.push_back(Elt: I->getOperand(i: 1));
228 Cond.push_back(Elt: I->getOperand(i: 2));
229 } else if (I->isReturn()) {
230 // Returns can't be analyzed, but we should run cleanup.
231 CantAnalyze = true;
232 } else if (I->getOpcode() == ARM::t2LoopEnd &&
233 MBB.getParent()
234 ->getSubtarget<ARMSubtarget>()
235 .enableMachinePipeliner()) {
236 if (!Cond.empty())
237 return true;
238 FBB = TBB;
239 TBB = I->getOperand(i: 1).getMBB();
240 Cond.push_back(Elt: MachineOperand::CreateImm(Val: I->getOpcode()));
241 Cond.push_back(Elt: I->getOperand(i: 0));
242 Cond.push_back(Elt: MachineOperand::CreateImm(Val: 0));
243 } else {
244 // We encountered other unrecognized terminator. Bail out immediately.
245 return true;
246 }
247
248 // Cleanup code - to be run for unpredicated unconditional branches and
249 // returns.
250 if (!isPredicated(MI: *I) &&
251 (isUncondBranchOpcode(Opc: I->getOpcode()) ||
252 isIndirectBranchOpcode(Opc: I->getOpcode()) ||
253 isJumpTableBranchOpcode(Opc: I->getOpcode()) ||
254 I->isReturn())) {
255 // Forget any previous condition branch information - it no longer applies.
256 Cond.clear();
257 FBB = nullptr;
258
259 // If we can modify the function, delete everything below this
260 // unconditional branch.
261 if (AllowModify) {
262 MachineBasicBlock::iterator DI = std::next(x: I);
263 while (DI != MBB.instr_end()) {
264 MachineInstr &InstToDelete = *DI;
265 ++DI;
266 // Speculation barriers must not be deleted.
267 if (isSpeculationBarrierEndBBOpcode(Opc: InstToDelete.getOpcode()))
268 continue;
269 InstToDelete.eraseFromParent();
270 }
271 }
272 }
273
274 if (CantAnalyze) {
275 // We may not be able to analyze the block, but we could still have
276 // an unconditional branch as the last instruction in the block, which
277 // just branches to layout successor. If this is the case, then just
278 // remove it if we're allowed to make modifications.
279 if (AllowModify && !isPredicated(MI: MBB.back()) &&
280 isUncondBranchOpcode(Opc: MBB.back().getOpcode()) &&
281 TBB && MBB.isLayoutSuccessor(MBB: TBB))
282 removeBranch(MBB);
283 return true;
284 }
285
286 if (I == MBB.instr_begin())
287 return false;
288
289 --I;
290 }
291
292 // We made it past the terminators without bailing out - we must have
293 // analyzed this branch successfully.
294 return false;
295}
296
297unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
298 int *BytesRemoved) const {
299 assert(!BytesRemoved && "code size not handled");
300
301 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
302 if (I == MBB.end())
303 return 0;
304
305 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
306 !isCondBranchOpcode(Opc: I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
307 return 0;
308
309 // Remove the branch.
310 I->eraseFromParent();
311
312 I = MBB.end();
313
314 if (I == MBB.begin()) return 1;
315 --I;
316 if (!isCondBranchOpcode(Opc: I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
317 return 1;
318
319 // Remove the branch.
320 I->eraseFromParent();
321 return 2;
322}
323
324unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
325 MachineBasicBlock *TBB,
326 MachineBasicBlock *FBB,
327 ArrayRef<MachineOperand> Cond,
328 const DebugLoc &DL,
329 int *BytesAdded) const {
330 assert(!BytesAdded && "code size not handled");
331 ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
332 int BOpc = !AFI->isThumbFunction()
333 ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
334 int BccOpc = !AFI->isThumbFunction()
335 ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
336 bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
337
338 // Shouldn't be a fall through.
339 assert(TBB && "insertBranch must not be told to insert a fallthrough");
340 assert((Cond.size() == 2 || Cond.size() == 0 || Cond.size() == 3) &&
341 "ARM branch conditions have two or three components!");
342
343 // For conditional branches, we use addOperand to preserve CPSR flags.
344
345 if (!FBB) {
346 if (Cond.empty()) { // Unconditional branch?
347 if (isThumb)
348 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: TBB).add(MOs: predOps(Pred: ARMCC::AL));
349 else
350 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: TBB);
351 } else if (Cond.size() == 2) {
352 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BccOpc))
353 .addMBB(MBB: TBB)
354 .addImm(Val: Cond[0].getImm())
355 .add(MO: Cond[1]);
356 } else
357 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm())).add(MO: Cond[1]).addMBB(MBB: TBB);
358 return 1;
359 }
360
361 // Two-way conditional branch.
362 if (Cond.size() == 2)
363 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BccOpc))
364 .addMBB(MBB: TBB)
365 .addImm(Val: Cond[0].getImm())
366 .add(MO: Cond[1]);
367 else if (Cond.size() == 3)
368 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm())).add(MO: Cond[1]).addMBB(MBB: TBB);
369 if (isThumb)
370 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: FBB).add(MOs: predOps(Pred: ARMCC::AL));
371 else
372 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: BOpc)).addMBB(MBB: FBB);
373 return 2;
374}
375
376bool ARMBaseInstrInfo::
377reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
378 if (Cond.size() == 2) {
379 ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
380 Cond[0].setImm(ARMCC::getOppositeCondition(CC));
381 return false;
382 }
383 return true;
384}
385
386bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
387 if (MI.isBundle()) {
388 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
389 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
390 while (++I != E && I->isInsideBundle()) {
391 int PIdx = I->findFirstPredOperandIdx();
392 if (PIdx != -1 && I->getOperand(i: PIdx).getImm() != ARMCC::AL)
393 return true;
394 }
395 return false;
396 }
397
398 int PIdx = MI.findFirstPredOperandIdx();
399 return PIdx != -1 && MI.getOperand(i: PIdx).getImm() != ARMCC::AL;
400}
401
402std::string ARMBaseInstrInfo::createMIROperandComment(
403 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
404 const TargetRegisterInfo *TRI) const {
405
406 // First, let's see if there is a generic comment for this operand
407 std::string GenericComment =
408 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
409 if (!GenericComment.empty())
410 return GenericComment;
411
412 // If not, check if we have an immediate operand.
413 if (!Op.isImm())
414 return std::string();
415
416 // And print its corresponding condition code if the immediate is a
417 // predicate.
418 int FirstPredOp = MI.findFirstPredOperandIdx();
419 if (FirstPredOp != (int) OpIdx)
420 return std::string();
421
422 std::string CC = "CC::";
423 CC += ARMCondCodeToString(CC: (ARMCC::CondCodes)Op.getImm());
424 return CC;
425}
426
427bool ARMBaseInstrInfo::PredicateInstruction(
428 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
429 unsigned Opc = MI.getOpcode();
430 if (isUncondBranchOpcode(Opc)) {
431 MI.setDesc(get(Opcode: getMatchingCondBranchOpcode(Opc)));
432 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
433 .addImm(Val: Pred[0].getImm())
434 .addReg(RegNo: Pred[1].getReg());
435 return true;
436 }
437
438 int PIdx = MI.findFirstPredOperandIdx();
439 if (PIdx != -1) {
440 MachineOperand &PMO = MI.getOperand(i: PIdx);
441 PMO.setImm(Pred[0].getImm());
442 MI.getOperand(i: PIdx+1).setReg(Pred[1].getReg());
443
444 // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
445 // IT block. This affects how they are printed.
446 const MCInstrDesc &MCID = MI.getDesc();
447 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
448 assert(MCID.operands()[1].isOptionalDef() &&
449 "CPSR def isn't expected operand");
450 assert((MI.getOperand(1).isDead() ||
451 MI.getOperand(1).getReg() != ARM::CPSR) &&
452 "if conversion tried to stop defining used CPSR");
453 MI.getOperand(i: 1).setReg(ARM::NoRegister);
454 }
455
456 return true;
457 }
458 return false;
459}
460
461bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
462 ArrayRef<MachineOperand> Pred2) const {
463 if (Pred1.size() > 2 || Pred2.size() > 2)
464 return false;
465
466 ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
467 ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
468 if (CC1 == CC2)
469 return true;
470
471 switch (CC1) {
472 default:
473 return false;
474 case ARMCC::AL:
475 return true;
476 case ARMCC::HS:
477 return CC2 == ARMCC::HI;
478 case ARMCC::LS:
479 return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
480 case ARMCC::GE:
481 return CC2 == ARMCC::GT;
482 case ARMCC::LE:
483 return CC2 == ARMCC::LT;
484 }
485}
486
487bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI,
488 std::vector<MachineOperand> &Pred,
489 bool SkipDead) const {
490 bool Found = false;
491 for (const MachineOperand &MO : MI.operands()) {
492 bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(PhysReg: ARM::CPSR);
493 bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR;
494 if (ClobbersCPSR || IsCPSR) {
495
496 // Filter out T1 instructions that have a dead CPSR,
497 // allowing IT blocks to be generated containing T1 instructions
498 const MCInstrDesc &MCID = MI.getDesc();
499 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() &&
500 SkipDead)
501 continue;
502
503 Pred.push_back(x: MO);
504 Found = true;
505 }
506 }
507
508 return Found;
509}
510
511bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
512 for (const auto &MO : MI.operands())
513 if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
514 return true;
515 return false;
516}
517
518static bool isEligibleForITBlock(const MachineInstr *MI) {
519 switch (MI->getOpcode()) {
520 default: return true;
521 case ARM::tADC: // ADC (register) T1
522 case ARM::tADDi3: // ADD (immediate) T1
523 case ARM::tADDi8: // ADD (immediate) T2
524 case ARM::tADDrr: // ADD (register) T1
525 case ARM::tAND: // AND (register) T1
526 case ARM::tASRri: // ASR (immediate) T1
527 case ARM::tASRrr: // ASR (register) T1
528 case ARM::tBIC: // BIC (register) T1
529 case ARM::tEOR: // EOR (register) T1
530 case ARM::tLSLri: // LSL (immediate) T1
531 case ARM::tLSLrr: // LSL (register) T1
532 case ARM::tLSRri: // LSR (immediate) T1
533 case ARM::tLSRrr: // LSR (register) T1
534 case ARM::tMUL: // MUL T1
535 case ARM::tMVN: // MVN (register) T1
536 case ARM::tORR: // ORR (register) T1
537 case ARM::tROR: // ROR (register) T1
538 case ARM::tRSB: // RSB (immediate) T1
539 case ARM::tSBC: // SBC (register) T1
540 case ARM::tSUBi3: // SUB (immediate) T1
541 case ARM::tSUBi8: // SUB (immediate) T2
542 case ARM::tSUBrr: // SUB (register) T1
543 return !ARMBaseInstrInfo::isCPSRDefined(MI: *MI);
544 }
545}
546
547/// isPredicable - Return true if the specified instruction can be predicated.
548/// By default, this returns true for every instruction with a
549/// PredicateOperand.
550bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
551 if (!MI.isPredicable())
552 return false;
553
554 if (MI.isBundle())
555 return false;
556
557 if (!isEligibleForITBlock(MI: &MI))
558 return false;
559
560 const MachineFunction *MF = MI.getParent()->getParent();
561 const ARMFunctionInfo *AFI =
562 MF->getInfo<ARMFunctionInfo>();
563
564 // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
565 // In their ARM encoding, they can't be encoded in a conditional form.
566 if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
567 return false;
568
569 // Make indirect control flow changes unpredictable when SLS mitigation is
570 // enabled.
571 const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>();
572 if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI))
573 return false;
574 if (ST.hardenSlsBlr() && isIndirectCall(MI))
575 return false;
576
577 if (AFI->isThumb2Function()) {
578 if (getSubtarget().restrictIT())
579 return isV8EligibleForIT(Instr: &MI);
580 }
581
582 return true;
583}
584
585namespace llvm {
586
587template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
588 for (const MachineOperand &MO : MI->operands()) {
589 if (!MO.isReg() || MO.isUndef() || MO.isUse())
590 continue;
591 if (MO.getReg() != ARM::CPSR)
592 continue;
593 if (!MO.isDead())
594 return false;
595 }
596 // all definitions of CPSR are dead
597 return true;
598}
599
600} // end namespace llvm
601
602/// GetInstSize - Return the size of the specified MachineInstr.
603///
604unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
605 const MachineBasicBlock &MBB = *MI.getParent();
606 const MachineFunction *MF = MBB.getParent();
607 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
608
609 const MCInstrDesc &MCID = MI.getDesc();
610
611 switch (MI.getOpcode()) {
612 default:
613 // Return the size specified in .td file. If there's none, return 0, as we
614 // can't define a default size (Thumb1 instructions are 2 bytes, Thumb2
615 // instructions are 2-4 bytes, and ARM instructions are 4 bytes), in
616 // contrast to AArch64 instructions which have a default size of 4 bytes for
617 // example.
618 return MCID.getSize();
619 case TargetOpcode::BUNDLE:
620 return getInstBundleSize(MI);
621 case TargetOpcode::COPY:
622 if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
623 return 4;
624 else
625 return 2;
626 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
627 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
628 case TargetOpcode::PATCHABLE_TAIL_CALL:
629 // Size of xray sled: Branch + 6 nops.
630 return 28;
631 case ARM::CONSTPOOL_ENTRY:
632 case ARM::JUMPTABLE_INSTS:
633 case ARM::JUMPTABLE_ADDRS:
634 case ARM::JUMPTABLE_TBB:
635 case ARM::JUMPTABLE_TBH:
636 // If this machine instr is a constant pool entry, its size is recorded as
637 // operand #2.
638 return MI.getOperand(i: 2).getImm();
639 case ARM::SPACE:
640 return MI.getOperand(i: 1).getImm();
641 case ARM::INLINEASM:
642 case ARM::INLINEASM_BR: {
643 // If this machine instr is an inline asm, measure it.
644 unsigned Size = getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
645 if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
646 Size = alignTo(Value: Size, Align: 4);
647 return Size;
648 }
649 case ARM::Int_eh_sjlj_longjmp:
650 return Subtarget.isTargetDarwin() || Subtarget.isTargetWindows() ? 16 : 20;
651 case ARM::tInt_eh_sjlj_longjmp:
652 return Subtarget.isTargetDarwin() || Subtarget.isTargetWindows() ? 10 : 12;
653 }
654}
655
656void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
657 MachineBasicBlock::iterator I,
658 MCRegister DestReg, bool KillSrc,
659 const ARMSubtarget &Subtarget) const {
660 unsigned Opc = Subtarget.isThumb()
661 ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
662 : ARM::MRS;
663
664 MachineInstrBuilder MIB =
665 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc), DestReg);
666
667 // There is only 1 A/R class MRS instruction, and it always refers to
668 // APSR. However, there are lots of other possibilities on M-class cores.
669 if (Subtarget.isMClass())
670 MIB.addImm(Val: 0x800);
671
672 MIB.add(MOs: predOps(Pred: ARMCC::AL))
673 .addReg(RegNo: ARM::CPSR, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
674}
675
676void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
677 MachineBasicBlock::iterator I,
678 MCRegister SrcReg, bool KillSrc,
679 const ARMSubtarget &Subtarget) const {
680 unsigned Opc = Subtarget.isThumb()
681 ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
682 : ARM::MSR;
683
684 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc));
685
686 if (Subtarget.isMClass())
687 MIB.addImm(Val: 0x800);
688 else
689 MIB.addImm(Val: 8);
690
691 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
692 .add(MOs: predOps(Pred: ARMCC::AL))
693 .addReg(RegNo: ARM::CPSR, Flags: RegState::Implicit | RegState::Define);
694}
695
696void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
697 MIB.addImm(Val: ARMVCC::None);
698 MIB.addReg(RegNo: 0);
699 MIB.addReg(RegNo: 0); // tp_reg
700}
701
702void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
703 Register DestReg) {
704 addUnpredicatedMveVpredNOp(MIB);
705 MIB.addReg(RegNo: DestReg, Flags: RegState::Undef);
706}
707
708void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
709 MIB.addImm(Val: Cond);
710 MIB.addReg(RegNo: ARM::VPR, Flags: RegState::Implicit);
711 MIB.addReg(RegNo: 0); // tp_reg
712}
713
714void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
715 unsigned Cond, unsigned Inactive) {
716 addPredicatedMveVpredNOp(MIB, Cond);
717 MIB.addReg(RegNo: Inactive);
718}
719
720void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
721 MachineBasicBlock::iterator I,
722 const DebugLoc &DL, Register DestReg,
723 Register SrcReg, bool KillSrc,
724 bool RenamableDest,
725 bool RenamableSrc) const {
726 bool GPRDest = ARM::GPRRegClass.contains(Reg: DestReg);
727 bool GPRSrc = ARM::GPRRegClass.contains(Reg: SrcReg);
728
729 if (GPRDest && GPRSrc) {
730 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MOVr), DestReg)
731 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
732 .add(MOs: predOps(Pred: ARMCC::AL))
733 .add(MO: condCodeOp());
734 return;
735 }
736
737 bool SPRDest = ARM::SPRRegClass.contains(Reg: DestReg);
738 bool SPRSrc = ARM::SPRRegClass.contains(Reg: SrcReg);
739
740 unsigned Opc = 0;
741 if (SPRDest && SPRSrc)
742 Opc = ARM::VMOVS;
743 else if (GPRDest && SPRSrc)
744 Opc = ARM::VMOVRS;
745 else if (SPRDest && GPRSrc)
746 Opc = ARM::VMOVSR;
747 else if (ARM::DPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg) && Subtarget.hasFP64())
748 Opc = ARM::VMOVD;
749 else if (ARM::QPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg))
750 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MQPRCopy;
751
752 if (Opc) {
753 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg);
754 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
755 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
756 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
757 if (Opc == ARM::MVE_VORR)
758 addUnpredicatedMveVpredROp(MIB, DestReg);
759 else if (Opc != ARM::MQPRCopy)
760 MIB.add(MOs: predOps(Pred: ARMCC::AL));
761 return;
762 }
763
764 // Handle register classes that require multiple instructions.
765 unsigned BeginIdx = 0;
766 unsigned SubRegs = 0;
767 int Spacing = 1;
768
769 // Use VORRq when possible.
770 if (ARM::QQPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
771 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
772 BeginIdx = ARM::qsub_0;
773 SubRegs = 2;
774 } else if (ARM::QQQQPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
775 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
776 BeginIdx = ARM::qsub_0;
777 SubRegs = 4;
778 // Fall back to VMOVD.
779 } else if (ARM::DPairRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
780 Opc = ARM::VMOVD;
781 BeginIdx = ARM::dsub_0;
782 SubRegs = 2;
783 } else if (ARM::DTripleRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
784 Opc = ARM::VMOVD;
785 BeginIdx = ARM::dsub_0;
786 SubRegs = 3;
787 } else if (ARM::DQuadRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
788 Opc = ARM::VMOVD;
789 BeginIdx = ARM::dsub_0;
790 SubRegs = 4;
791 } else if (ARM::GPRPairRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
792 Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
793 BeginIdx = ARM::gsub_0;
794 SubRegs = 2;
795 } else if (ARM::DPairSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
796 Opc = ARM::VMOVD;
797 BeginIdx = ARM::dsub_0;
798 SubRegs = 2;
799 Spacing = 2;
800 } else if (ARM::DTripleSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
801 Opc = ARM::VMOVD;
802 BeginIdx = ARM::dsub_0;
803 SubRegs = 3;
804 Spacing = 2;
805 } else if (ARM::DQuadSpcRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
806 Opc = ARM::VMOVD;
807 BeginIdx = ARM::dsub_0;
808 SubRegs = 4;
809 Spacing = 2;
810 } else if (ARM::DPRRegClass.contains(Reg1: DestReg, Reg2: SrcReg) &&
811 !Subtarget.hasFP64()) {
812 Opc = ARM::VMOVS;
813 BeginIdx = ARM::ssub_0;
814 SubRegs = 2;
815 } else if (SrcReg == ARM::CPSR) {
816 copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
817 return;
818 } else if (DestReg == ARM::CPSR) {
819 copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
820 return;
821 } else if (DestReg == ARM::VPR) {
822 assert(ARM::GPRRegClass.contains(SrcReg));
823 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMSR_P0), DestReg)
824 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
825 .add(MOs: predOps(Pred: ARMCC::AL));
826 return;
827 } else if (SrcReg == ARM::VPR) {
828 assert(ARM::GPRRegClass.contains(DestReg));
829 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMRS_P0), DestReg)
830 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
831 .add(MOs: predOps(Pred: ARMCC::AL));
832 return;
833 } else if (DestReg == ARM::FPSCR_NZCV) {
834 assert(ARM::GPRRegClass.contains(SrcReg));
835 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMSR_FPSCR_NZCVQC), DestReg)
836 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
837 .add(MOs: predOps(Pred: ARMCC::AL));
838 return;
839 } else if (SrcReg == ARM::FPSCR_NZCV) {
840 assert(ARM::GPRRegClass.contains(DestReg));
841 BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: ARM::VMRS_FPSCR_NZCVQC), DestReg)
842 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
843 .add(MOs: predOps(Pred: ARMCC::AL));
844 return;
845 }
846
847 assert(Opc && "Impossible reg-to-reg copy");
848
849 const TargetRegisterInfo *TRI = &getRegisterInfo();
850 MachineInstrBuilder Mov;
851
852 // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
853 if (TRI->regsOverlap(RegA: SrcReg, RegB: TRI->getSubReg(Reg: DestReg, Idx: BeginIdx))) {
854 BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
855 Spacing = -Spacing;
856 }
857#ifndef NDEBUG
858 SmallSet<unsigned, 4> DstRegs;
859#endif
860 for (unsigned i = 0; i != SubRegs; ++i) {
861 Register Dst = TRI->getSubReg(Reg: DestReg, Idx: BeginIdx + i * Spacing);
862 Register Src = TRI->getSubReg(Reg: SrcReg, Idx: BeginIdx + i * Spacing);
863 assert(Dst && Src && "Bad sub-register");
864#ifndef NDEBUG
865 assert(!DstRegs.count(Src) && "destructive vector copy");
866 DstRegs.insert(Dst);
867#endif
868 Mov = BuildMI(BB&: MBB, I, MIMD: I->getDebugLoc(), MCID: get(Opcode: Opc), DestReg: Dst).addReg(RegNo: Src);
869 // VORR (NEON or MVE) takes two source operands.
870 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
871 Mov.addReg(RegNo: Src);
872 }
873 // MVE VORR takes predicate operands in place of an ordinary condition.
874 if (Opc == ARM::MVE_VORR)
875 addUnpredicatedMveVpredROp(MIB&: Mov, DestReg: Dst);
876 else
877 Mov = Mov.add(MOs: predOps(Pred: ARMCC::AL));
878 // MOVr can set CC.
879 if (Opc == ARM::MOVr)
880 Mov = Mov.add(MO: condCodeOp());
881 }
882 // Add implicit super-register defs and kills to the last instruction.
883 Mov->addRegisterDefined(Reg: DestReg, RegInfo: TRI);
884 if (KillSrc)
885 Mov->addRegisterKilled(IncomingReg: SrcReg, RegInfo: TRI);
886}
887
888std::optional<DestSourcePair>
889ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
890 // VMOVRRD is also a copy instruction but it requires
891 // special way of handling. It is more complex copy version
892 // and since that we are not considering it. For recognition
893 // of such instruction isExtractSubregLike MI interface function
894 // could be used.
895 // VORRq is considered as a move only if two inputs are
896 // the same register.
897 if (!MI.isMoveReg() ||
898 (MI.getOpcode() == ARM::VORRq &&
899 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg()))
900 return std::nullopt;
901 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
902}
903
904std::optional<ParamLoadedValue>
905ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
906 Register Reg) const {
907 if (auto DstSrcPair = isCopyInstrImpl(MI)) {
908 Register DstReg = DstSrcPair->Destination->getReg();
909
910 // TODO: We don't handle cases where the forwarding reg is narrower/wider
911 // than the copy registers. Consider for example:
912 //
913 // s16 = VMOVS s0
914 // s17 = VMOVS s1
915 // call @callee(d0)
916 //
917 // We'd like to describe the call site value of d0 as d8, but this requires
918 // gathering and merging the descriptions for the two VMOVS instructions.
919 //
920 // We also don't handle the reverse situation, where the forwarding reg is
921 // narrower than the copy destination:
922 //
923 // d8 = VMOVD d0
924 // call @callee(s1)
925 //
926 // We need to produce a fragment description (the call site value of s1 is
927 // /not/ just d8).
928 if (DstReg != Reg)
929 return std::nullopt;
930 }
931 return TargetInstrInfo::describeLoadedValue(MI, Reg);
932}
933
934const MachineInstrBuilder &ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB,
935 unsigned Reg,
936 unsigned SubIdx,
937 RegState State) const {
938 if (!SubIdx)
939 return MIB.addReg(RegNo: Reg, Flags: State);
940
941 if (Register::isPhysicalRegister(Reg))
942 return MIB.addReg(RegNo: getRegisterInfo().getSubReg(Reg, Idx: SubIdx), Flags: State);
943 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
944}
945
946void ARMBaseInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
947 MachineBasicBlock::iterator I,
948 Register SrcReg, bool isKill, int FI,
949 const TargetRegisterClass *RC,
950 Register VReg,
951 MachineInstr::MIFlag Flags) const {
952 MachineFunction &MF = *MBB.getParent();
953 MachineFrameInfo &MFI = MF.getFrameInfo();
954 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
955 const ARMBaseRegisterInfo &TRI = getRegisterInfo();
956
957 MachineMemOperand *MMO = MF.getMachineMemOperand(
958 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI), F: MachineMemOperand::MOStore,
959 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
960
961 switch (TRI.getSpillSize(RC: *RC)) {
962 case 2:
963 if (ARM::HPRRegClass.hasSubClassEq(RC)) {
964 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRH))
965 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
966 .addFrameIndex(Idx: FI)
967 .addImm(Val: 0)
968 .addMemOperand(MMO)
969 .add(MOs: predOps(Pred: ARMCC::AL));
970 } else
971 llvm_unreachable("Unknown reg class!");
972 break;
973 case 4:
974 if (ARM::GPRRegClass.hasSubClassEq(RC)) {
975 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STRi12))
976 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
977 .addFrameIndex(Idx: FI)
978 .addImm(Val: 0)
979 .addMemOperand(MMO)
980 .add(MOs: predOps(Pred: ARMCC::AL));
981 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
982 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRS))
983 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
984 .addFrameIndex(Idx: FI)
985 .addImm(Val: 0)
986 .addMemOperand(MMO)
987 .add(MOs: predOps(Pred: ARMCC::AL));
988 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
989 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTR_P0_off))
990 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
991 .addFrameIndex(Idx: FI)
992 .addImm(Val: 0)
993 .addMemOperand(MMO)
994 .add(MOs: predOps(Pred: ARMCC::AL));
995 } else if (ARM::cl_FPSCR_NZCVRegClass.hasSubClassEq(RC)) {
996 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTR_FPSCR_NZCVQC_off))
997 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
998 .addFrameIndex(Idx: FI)
999 .addImm(Val: 0)
1000 .addMemOperand(MMO)
1001 .add(MOs: predOps(Pred: ARMCC::AL));
1002 } else
1003 llvm_unreachable("Unknown reg class!");
1004 break;
1005 case 8:
1006 if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1007 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTRD))
1008 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1009 .addFrameIndex(Idx: FI)
1010 .addImm(Val: 0)
1011 .addMemOperand(MMO)
1012 .add(MOs: predOps(Pred: ARMCC::AL));
1013 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1014 if (Subtarget.hasV5TEOps()) {
1015 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STRD));
1016 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_0, State: getKillRegState(B: isKill));
1017 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_1, State: {});
1018 MIB.addFrameIndex(Idx: FI).addReg(RegNo: 0).addImm(Val: 0).addMemOperand(MMO)
1019 .add(MOs: predOps(Pred: ARMCC::AL));
1020 } else {
1021 // Fallback to STM instruction, which has existed since the dawn of
1022 // time.
1023 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::STMIA))
1024 .addFrameIndex(Idx: FI)
1025 .addMemOperand(MMO)
1026 .add(MOs: predOps(Pred: ARMCC::AL));
1027 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_0, State: getKillRegState(B: isKill));
1028 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::gsub_1, State: {});
1029 }
1030 } else
1031 llvm_unreachable("Unknown reg class!");
1032 break;
1033 case 16:
1034 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1035 // Use aligned spills if the stack can be realigned.
1036 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1037 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1q64))
1038 .addFrameIndex(Idx: FI)
1039 .addImm(Val: 16)
1040 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1041 .addMemOperand(MMO)
1042 .add(MOs: predOps(Pred: ARMCC::AL));
1043 } else {
1044 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTMQIA))
1045 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1046 .addFrameIndex(Idx: FI)
1047 .addMemOperand(MMO)
1048 .add(MOs: predOps(Pred: ARMCC::AL));
1049 }
1050 } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1051 Subtarget.hasMVEIntegerOps()) {
1052 auto MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MVE_VSTRWU32));
1053 MIB.addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1054 .addFrameIndex(Idx: FI)
1055 .addImm(Val: 0)
1056 .addMemOperand(MMO);
1057 addUnpredicatedMveVpredNOp(MIB);
1058 } else
1059 llvm_unreachable("Unknown reg class!");
1060 break;
1061 case 24:
1062 if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1063 // Use aligned spills if the stack can be realigned.
1064 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1065 Subtarget.hasNEON()) {
1066 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1d64TPseudo))
1067 .addFrameIndex(Idx: FI)
1068 .addImm(Val: 16)
1069 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1070 .addMemOperand(MMO)
1071 .add(MOs: predOps(Pred: ARMCC::AL));
1072 } else {
1073 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(),
1074 MCID: get(Opcode: ARM::VSTMDIA))
1075 .addFrameIndex(Idx: FI)
1076 .add(MOs: predOps(Pred: ARMCC::AL))
1077 .addMemOperand(MMO);
1078 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1079 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1080 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1081 }
1082 } else
1083 llvm_unreachable("Unknown reg class!");
1084 break;
1085 case 32:
1086 if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1087 ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1088 ARM::DQuadRegClass.hasSubClassEq(RC)) {
1089 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1090 Subtarget.hasNEON()) {
1091 // FIXME: It's possible to only store part of the QQ register if the
1092 // spilled def has a sub-register index.
1093 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VST1d64QPseudo))
1094 .addFrameIndex(Idx: FI)
1095 .addImm(Val: 16)
1096 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1097 .addMemOperand(MMO)
1098 .add(MOs: predOps(Pred: ARMCC::AL));
1099 } else if (Subtarget.hasMVEIntegerOps()) {
1100 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MQQPRStore))
1101 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1102 .addFrameIndex(Idx: FI)
1103 .addMemOperand(MMO);
1104 } else {
1105 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(),
1106 MCID: get(Opcode: ARM::VSTMDIA))
1107 .addFrameIndex(Idx: FI)
1108 .add(MOs: predOps(Pred: ARMCC::AL))
1109 .addMemOperand(MMO);
1110 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1111 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1112 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1113 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_3, State: {});
1114 }
1115 } else
1116 llvm_unreachable("Unknown reg class!");
1117 break;
1118 case 64:
1119 if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1120 Subtarget.hasMVEIntegerOps()) {
1121 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::MQQQQPRStore))
1122 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
1123 .addFrameIndex(Idx: FI)
1124 .addMemOperand(MMO);
1125 } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1126 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode: ARM::VSTMDIA))
1127 .addFrameIndex(Idx: FI)
1128 .add(MOs: predOps(Pred: ARMCC::AL))
1129 .addMemOperand(MMO);
1130 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_0, State: getKillRegState(B: isKill));
1131 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_1, State: {});
1132 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_2, State: {});
1133 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_3, State: {});
1134 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_4, State: {});
1135 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_5, State: {});
1136 MIB = AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_6, State: {});
1137 AddDReg(MIB, Reg: SrcReg, SubIdx: ARM::dsub_7, State: {});
1138 } else
1139 llvm_unreachable("Unknown reg class!");
1140 break;
1141 default:
1142 llvm_unreachable("Unknown reg class!");
1143 }
1144}
1145
1146Register ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1147 int &FrameIndex) const {
1148 switch (MI.getOpcode()) {
1149 default: break;
1150 case ARM::STRrs:
1151 case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1152 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isReg() &&
1153 MI.getOperand(i: 3).isImm() && MI.getOperand(i: 2).getReg() == 0 &&
1154 MI.getOperand(i: 3).getImm() == 0) {
1155 FrameIndex = MI.getOperand(i: 1).getIndex();
1156 return MI.getOperand(i: 0).getReg();
1157 }
1158 break;
1159 case ARM::STRi12:
1160 case ARM::t2STRi12:
1161 case ARM::tSTRspi:
1162 case ARM::VSTRD:
1163 case ARM::VSTRS:
1164 case ARM::VSTRH:
1165 case ARM::VSTR_P0_off:
1166 case ARM::VSTR_FPSCR_NZCVQC_off:
1167 case ARM::MVE_VSTRWU32:
1168 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
1169 MI.getOperand(i: 2).getImm() == 0) {
1170 FrameIndex = MI.getOperand(i: 1).getIndex();
1171 return MI.getOperand(i: 0).getReg();
1172 }
1173 break;
1174 case ARM::VST1q64:
1175 case ARM::VST1d64TPseudo:
1176 case ARM::VST1d64QPseudo:
1177 if (MI.getOperand(i: 0).isFI() && MI.getOperand(i: 2).getSubReg() == 0) {
1178 FrameIndex = MI.getOperand(i: 0).getIndex();
1179 return MI.getOperand(i: 2).getReg();
1180 }
1181 break;
1182 case ARM::VSTMQIA:
1183 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1184 FrameIndex = MI.getOperand(i: 1).getIndex();
1185 return MI.getOperand(i: 0).getReg();
1186 }
1187 break;
1188 case ARM::MQQPRStore:
1189 case ARM::MQQQQPRStore:
1190 if (MI.getOperand(i: 1).isFI()) {
1191 FrameIndex = MI.getOperand(i: 1).getIndex();
1192 return MI.getOperand(i: 0).getReg();
1193 }
1194 break;
1195 }
1196
1197 return 0;
1198}
1199
1200Register ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1201 int &FrameIndex) const {
1202 SmallVector<const MachineMemOperand *, 1> Accesses;
1203 if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1204 Accesses.size() == 1) {
1205 FrameIndex =
1206 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
1207 ->getFrameIndex();
1208 return true;
1209 }
1210 return false;
1211}
1212
1213void ARMBaseInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1214 MachineBasicBlock::iterator I,
1215 Register DestReg, int FI,
1216 const TargetRegisterClass *RC,
1217 Register VReg, unsigned SubReg,
1218 MachineInstr::MIFlag Flags) const {
1219 DebugLoc DL;
1220 if (I != MBB.end()) DL = I->getDebugLoc();
1221 MachineFunction &MF = *MBB.getParent();
1222 MachineFrameInfo &MFI = MF.getFrameInfo();
1223 const Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
1224 MachineMemOperand *MMO = MF.getMachineMemOperand(
1225 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI), F: MachineMemOperand::MOLoad,
1226 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
1227
1228 const ARMBaseRegisterInfo &TRI = getRegisterInfo();
1229 switch (TRI.getSpillSize(RC: *RC)) {
1230 case 2:
1231 if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1232 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRH), DestReg)
1233 .addFrameIndex(Idx: FI)
1234 .addImm(Val: 0)
1235 .addMemOperand(MMO)
1236 .add(MOs: predOps(Pred: ARMCC::AL));
1237 } else
1238 llvm_unreachable("Unknown reg class!");
1239 break;
1240 case 4:
1241 if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1242 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDRi12), DestReg)
1243 .addFrameIndex(Idx: FI)
1244 .addImm(Val: 0)
1245 .addMemOperand(MMO)
1246 .add(MOs: predOps(Pred: ARMCC::AL));
1247 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1248 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRS), DestReg)
1249 .addFrameIndex(Idx: FI)
1250 .addImm(Val: 0)
1251 .addMemOperand(MMO)
1252 .add(MOs: predOps(Pred: ARMCC::AL));
1253 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1254 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDR_P0_off), DestReg)
1255 .addFrameIndex(Idx: FI)
1256 .addImm(Val: 0)
1257 .addMemOperand(MMO)
1258 .add(MOs: predOps(Pred: ARMCC::AL));
1259 } else if (ARM::cl_FPSCR_NZCVRegClass.hasSubClassEq(RC)) {
1260 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDR_FPSCR_NZCVQC_off), DestReg)
1261 .addFrameIndex(Idx: FI)
1262 .addImm(Val: 0)
1263 .addMemOperand(MMO)
1264 .add(MOs: predOps(Pred: ARMCC::AL));
1265 } else
1266 llvm_unreachable("Unknown reg class!");
1267 break;
1268 case 8:
1269 if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1270 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDRD), DestReg)
1271 .addFrameIndex(Idx: FI)
1272 .addImm(Val: 0)
1273 .addMemOperand(MMO)
1274 .add(MOs: predOps(Pred: ARMCC::AL));
1275 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1276 MachineInstrBuilder MIB;
1277
1278 if (Subtarget.hasV5TEOps()) {
1279 MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDRD));
1280 AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_0, State: RegState::DefineNoRead);
1281 AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_1, State: RegState::DefineNoRead);
1282 MIB.addFrameIndex(Idx: FI).addReg(RegNo: 0).addImm(Val: 0).addMemOperand(MMO)
1283 .add(MOs: predOps(Pred: ARMCC::AL));
1284 } else {
1285 // Fallback to LDM instruction, which has existed since the dawn of
1286 // time.
1287 MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::LDMIA))
1288 .addFrameIndex(Idx: FI)
1289 .addMemOperand(MMO)
1290 .add(MOs: predOps(Pred: ARMCC::AL));
1291 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_0, State: RegState::DefineNoRead);
1292 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::gsub_1, State: RegState::DefineNoRead);
1293 }
1294
1295 if (DestReg.isPhysical())
1296 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1297 } else
1298 llvm_unreachable("Unknown reg class!");
1299 break;
1300 case 16:
1301 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1302 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1303 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1q64), DestReg)
1304 .addFrameIndex(Idx: FI)
1305 .addImm(Val: 16)
1306 .addMemOperand(MMO)
1307 .add(MOs: predOps(Pred: ARMCC::AL));
1308 } else {
1309 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMQIA), DestReg)
1310 .addFrameIndex(Idx: FI)
1311 .addMemOperand(MMO)
1312 .add(MOs: predOps(Pred: ARMCC::AL));
1313 }
1314 } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1315 Subtarget.hasMVEIntegerOps()) {
1316 auto MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MVE_VLDRWU32), DestReg);
1317 MIB.addFrameIndex(Idx: FI)
1318 .addImm(Val: 0)
1319 .addMemOperand(MMO);
1320 addUnpredicatedMveVpredNOp(MIB);
1321 } else
1322 llvm_unreachable("Unknown reg class!");
1323 break;
1324 case 24:
1325 if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1326 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1327 Subtarget.hasNEON()) {
1328 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1d64TPseudo), DestReg)
1329 .addFrameIndex(Idx: FI)
1330 .addImm(Val: 16)
1331 .addMemOperand(MMO)
1332 .add(MOs: predOps(Pred: ARMCC::AL));
1333 } else {
1334 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1335 .addFrameIndex(Idx: FI)
1336 .addMemOperand(MMO)
1337 .add(MOs: predOps(Pred: ARMCC::AL));
1338 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1339 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1340 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1341 if (DestReg.isPhysical())
1342 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1343 }
1344 } else
1345 llvm_unreachable("Unknown reg class!");
1346 break;
1347 case 32:
1348 if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1349 ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1350 ARM::DQuadRegClass.hasSubClassEq(RC)) {
1351 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1352 Subtarget.hasNEON()) {
1353 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLD1d64QPseudo), DestReg)
1354 .addFrameIndex(Idx: FI)
1355 .addImm(Val: 16)
1356 .addMemOperand(MMO)
1357 .add(MOs: predOps(Pred: ARMCC::AL));
1358 } else if (Subtarget.hasMVEIntegerOps()) {
1359 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MQQPRLoad), DestReg)
1360 .addFrameIndex(Idx: FI)
1361 .addMemOperand(MMO);
1362 } else {
1363 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1364 .addFrameIndex(Idx: FI)
1365 .add(MOs: predOps(Pred: ARMCC::AL))
1366 .addMemOperand(MMO);
1367 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1368 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1369 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1370 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_3, State: RegState::DefineNoRead);
1371 if (DestReg.isPhysical())
1372 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1373 }
1374 } else
1375 llvm_unreachable("Unknown reg class!");
1376 break;
1377 case 64:
1378 if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1379 Subtarget.hasMVEIntegerOps()) {
1380 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::MQQQQPRLoad), DestReg)
1381 .addFrameIndex(Idx: FI)
1382 .addMemOperand(MMO);
1383 } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1384 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ARM::VLDMDIA))
1385 .addFrameIndex(Idx: FI)
1386 .add(MOs: predOps(Pred: ARMCC::AL))
1387 .addMemOperand(MMO);
1388 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_0, State: RegState::DefineNoRead);
1389 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_1, State: RegState::DefineNoRead);
1390 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_2, State: RegState::DefineNoRead);
1391 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_3, State: RegState::DefineNoRead);
1392 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_4, State: RegState::DefineNoRead);
1393 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_5, State: RegState::DefineNoRead);
1394 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_6, State: RegState::DefineNoRead);
1395 MIB = AddDReg(MIB, Reg: DestReg, SubIdx: ARM::dsub_7, State: RegState::DefineNoRead);
1396 if (DestReg.isPhysical())
1397 MIB.addReg(RegNo: DestReg, Flags: RegState::ImplicitDefine);
1398 } else
1399 llvm_unreachable("Unknown reg class!");
1400 break;
1401 default:
1402 llvm_unreachable("Unknown regclass!");
1403 }
1404}
1405
1406Register ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1407 int &FrameIndex) const {
1408 switch (MI.getOpcode()) {
1409 default: break;
1410 case ARM::LDRrs:
1411 case ARM::t2LDRs: // FIXME: don't use t2LDRs to access frame.
1412 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isReg() &&
1413 MI.getOperand(i: 3).isImm() && MI.getOperand(i: 2).getReg() == 0 &&
1414 MI.getOperand(i: 3).getImm() == 0) {
1415 FrameIndex = MI.getOperand(i: 1).getIndex();
1416 return MI.getOperand(i: 0).getReg();
1417 }
1418 break;
1419 case ARM::LDRi12:
1420 case ARM::t2LDRi12:
1421 case ARM::tLDRspi:
1422 case ARM::VLDRD:
1423 case ARM::VLDRS:
1424 case ARM::VLDRH:
1425 case ARM::VLDR_P0_off:
1426 case ARM::VLDR_FPSCR_NZCVQC_off:
1427 case ARM::MVE_VLDRWU32:
1428 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
1429 MI.getOperand(i: 2).getImm() == 0) {
1430 FrameIndex = MI.getOperand(i: 1).getIndex();
1431 return MI.getOperand(i: 0).getReg();
1432 }
1433 break;
1434 case ARM::VLD1q64:
1435 case ARM::VLD1d8TPseudo:
1436 case ARM::VLD1d16TPseudo:
1437 case ARM::VLD1d32TPseudo:
1438 case ARM::VLD1d64TPseudo:
1439 case ARM::VLD1d8QPseudo:
1440 case ARM::VLD1d16QPseudo:
1441 case ARM::VLD1d32QPseudo:
1442 case ARM::VLD1d64QPseudo:
1443 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1444 FrameIndex = MI.getOperand(i: 1).getIndex();
1445 return MI.getOperand(i: 0).getReg();
1446 }
1447 break;
1448 case ARM::VLDMQIA:
1449 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 0).getSubReg() == 0) {
1450 FrameIndex = MI.getOperand(i: 1).getIndex();
1451 return MI.getOperand(i: 0).getReg();
1452 }
1453 break;
1454 case ARM::MQQPRLoad:
1455 case ARM::MQQQQPRLoad:
1456 if (MI.getOperand(i: 1).isFI()) {
1457 FrameIndex = MI.getOperand(i: 1).getIndex();
1458 return MI.getOperand(i: 0).getReg();
1459 }
1460 break;
1461 }
1462
1463 return 0;
1464}
1465
1466Register ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1467 int &FrameIndex) const {
1468 SmallVector<const MachineMemOperand *, 1> Accesses;
1469 if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1470 Accesses.size() == 1) {
1471 FrameIndex =
1472 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
1473 ->getFrameIndex();
1474 return true;
1475 }
1476 return false;
1477}
1478
1479/// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1480/// depending on whether the result is used.
1481void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1482 bool isThumb1 = Subtarget.isThumb1Only();
1483 bool isThumb2 = Subtarget.isThumb2();
1484 const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1485
1486 DebugLoc dl = MI->getDebugLoc();
1487 MachineBasicBlock *BB = MI->getParent();
1488
1489 MachineInstrBuilder LDM, STM;
1490 if (isThumb1 || !MI->getOperand(i: 1).isDead()) {
1491 MachineOperand LDWb(MI->getOperand(i: 1));
1492 LDM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2LDMIA_UPD
1493 : isThumb1 ? ARM::tLDMIA_UPD
1494 : ARM::LDMIA_UPD))
1495 .add(MO: LDWb);
1496 } else {
1497 LDM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1498 }
1499
1500 if (isThumb1 || !MI->getOperand(i: 0).isDead()) {
1501 MachineOperand STWb(MI->getOperand(i: 0));
1502 STM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2STMIA_UPD
1503 : isThumb1 ? ARM::tSTMIA_UPD
1504 : ARM::STMIA_UPD))
1505 .add(MO: STWb);
1506 } else {
1507 STM = BuildMI(BB&: *BB, I: MI, MIMD: dl, MCID: TII->get(Opcode: isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1508 }
1509
1510 MachineOperand LDBase(MI->getOperand(i: 3));
1511 LDM.add(MO: LDBase).add(MOs: predOps(Pred: ARMCC::AL));
1512
1513 MachineOperand STBase(MI->getOperand(i: 2));
1514 STM.add(MO: STBase).add(MOs: predOps(Pred: ARMCC::AL));
1515
1516 // Sort the scratch registers into ascending order.
1517 const TargetRegisterInfo &TRI = getRegisterInfo();
1518 SmallVector<unsigned, 6> ScratchRegs;
1519 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 5))
1520 ScratchRegs.push_back(Elt: MO.getReg());
1521 llvm::sort(C&: ScratchRegs,
1522 Comp: [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1523 return TRI.getEncodingValue(Reg: Reg1) <
1524 TRI.getEncodingValue(Reg: Reg2);
1525 });
1526
1527 for (const auto &Reg : ScratchRegs) {
1528 LDM.addReg(RegNo: Reg, Flags: RegState::Define);
1529 STM.addReg(RegNo: Reg, Flags: RegState::Kill);
1530 }
1531
1532 BB->erase(I: MI);
1533}
1534
1535bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1536 if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1537 expandLoadStackGuard(MI);
1538 MI.getParent()->erase(I: MI);
1539 return true;
1540 }
1541
1542 if (MI.getOpcode() == ARM::MEMCPY) {
1543 expandMEMCPY(MI);
1544 return true;
1545 }
1546
1547 // This hook gets to expand COPY instructions before they become
1548 // copyPhysReg() calls. Look for VMOVS instructions that can legally be
1549 // widened to VMOVD. We prefer the VMOVD when possible because it may be
1550 // changed into a VORR that can go down the NEON pipeline.
1551 if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1552 return false;
1553
1554 // Look for a copy between even S-registers. That is where we keep floats
1555 // when using NEON v2f32 instructions for f32 arithmetic.
1556 Register DstRegS = MI.getOperand(i: 0).getReg();
1557 Register SrcRegS = MI.getOperand(i: 1).getReg();
1558 if (!ARM::SPRRegClass.contains(Reg1: DstRegS, Reg2: SrcRegS))
1559 return false;
1560
1561 const TargetRegisterInfo *TRI = &getRegisterInfo();
1562 MCRegister DstRegD =
1563 TRI->getMatchingSuperReg(Reg: DstRegS, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
1564 MCRegister SrcRegD =
1565 TRI->getMatchingSuperReg(Reg: SrcRegS, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
1566 if (!DstRegD || !SrcRegD)
1567 return false;
1568
1569 // We want to widen this into a DstRegD = VMOVD SrcRegD copy. This is only
1570 // legal if the COPY already defines the full DstRegD, and it isn't a
1571 // sub-register insertion.
1572 if (!MI.definesRegister(Reg: DstRegD, TRI) || MI.readsRegister(Reg: DstRegD, TRI))
1573 return false;
1574
1575 // A dead copy shouldn't show up here, but reject it just in case.
1576 if (MI.getOperand(i: 0).isDead())
1577 return false;
1578
1579 // All clear, widen the COPY.
1580 LLVM_DEBUG(dbgs() << "widening: " << MI);
1581 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1582
1583 // Get rid of the old implicit-def of DstRegD. Leave it if it defines a Q-reg
1584 // or some other super-register.
1585 int ImpDefIdx = MI.findRegisterDefOperandIdx(Reg: DstRegD, /*TRI=*/nullptr);
1586 if (ImpDefIdx != -1)
1587 MI.removeOperand(OpNo: ImpDefIdx);
1588
1589 // Change the opcode and operands.
1590 MI.setDesc(get(Opcode: ARM::VMOVD));
1591 MI.getOperand(i: 0).setReg(DstRegD);
1592 MI.getOperand(i: 1).setReg(SrcRegD);
1593 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1594
1595 // We are now reading SrcRegD instead of SrcRegS. This may upset the
1596 // register scavenger and machine verifier, so we need to indicate that we
1597 // are reading an undefined value from SrcRegD, but a proper value from
1598 // SrcRegS.
1599 MI.getOperand(i: 1).setIsUndef();
1600 MIB.addReg(RegNo: SrcRegS, Flags: RegState::Implicit);
1601
1602 // SrcRegD may actually contain an unrelated value in the ssub_1
1603 // sub-register. Don't kill it. Only kill the ssub_0 sub-register.
1604 if (MI.getOperand(i: 1).isKill()) {
1605 MI.getOperand(i: 1).setIsKill(false);
1606 MI.addRegisterKilled(IncomingReg: SrcRegS, RegInfo: TRI, AddIfNotFound: true);
1607 }
1608
1609 LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1610 return true;
1611}
1612
1613/// Create a copy of a const pool value. Update CPI to the new index and return
1614/// the label UID.
1615static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1616 MachineConstantPool *MCP = MF.getConstantPool();
1617 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1618
1619 const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1620 assert(MCPE.isMachineConstantPoolEntry() &&
1621 "Expecting a machine constantpool entry!");
1622 ARMConstantPoolValue *ACPV =
1623 static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1624
1625 unsigned PCLabelId = AFI->createPICLabelUId();
1626 ARMConstantPoolValue *NewCPV = nullptr;
1627
1628 // FIXME: The below assumes PIC relocation model and that the function
1629 // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1630 // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1631 // instructions, so that's probably OK, but is PIC always correct when
1632 // we get here?
1633 if (ACPV->isGlobalValue())
1634 NewCPV = ARMConstantPoolConstant::Create(
1635 C: cast<ARMConstantPoolConstant>(Val: ACPV)->getGV(), ID: PCLabelId, Kind: ARMCP::CPValue,
1636 PCAdj: 4, Modifier: ACPV->getModifier(), AddCurrentAddress: ACPV->mustAddCurrentAddress());
1637 else if (ACPV->isExtSymbol())
1638 NewCPV = ARMConstantPoolSymbol::
1639 Create(C&: MF.getFunction().getContext(),
1640 s: cast<ARMConstantPoolSymbol>(Val: ACPV)->getSymbol(), ID: PCLabelId, PCAdj: 4);
1641 else if (ACPV->isBlockAddress())
1642 NewCPV = ARMConstantPoolConstant::
1643 Create(C: cast<ARMConstantPoolConstant>(Val: ACPV)->getBlockAddress(), ID: PCLabelId,
1644 Kind: ARMCP::CPBlockAddress, PCAdj: 4);
1645 else if (ACPV->isLSDA())
1646 NewCPV = ARMConstantPoolConstant::Create(C: &MF.getFunction(), ID: PCLabelId,
1647 Kind: ARMCP::CPLSDA, PCAdj: 4);
1648 else if (ACPV->isMachineBasicBlock())
1649 NewCPV = ARMConstantPoolMBB::
1650 Create(C&: MF.getFunction().getContext(),
1651 mbb: cast<ARMConstantPoolMBB>(Val: ACPV)->getMBB(), ID: PCLabelId, PCAdj: 4);
1652 else
1653 llvm_unreachable("Unexpected ARM constantpool value type!!");
1654 CPI = MCP->getConstantPoolIndex(V: NewCPV, Alignment: MCPE.getAlign());
1655 return PCLabelId;
1656}
1657
1658void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1659 MachineBasicBlock::iterator I,
1660 Register DestReg, unsigned SubIdx,
1661 const MachineInstr &Orig,
1662 LaneBitmask UsedLanes) const {
1663 unsigned Opcode = Orig.getOpcode();
1664 switch (Opcode) {
1665 default: {
1666 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
1667 MI->substituteRegister(FromReg: Orig.getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
1668 MBB.insert(I, MI);
1669 break;
1670 }
1671 case ARM::tLDRpci_pic:
1672 case ARM::t2LDRpci_pic: {
1673 MachineFunction &MF = *MBB.getParent();
1674 unsigned CPI = Orig.getOperand(i: 1).getIndex();
1675 unsigned PCLabelId = duplicateCPV(MF, CPI);
1676 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode), DestReg)
1677 .addConstantPoolIndex(Idx: CPI)
1678 .addImm(Val: PCLabelId)
1679 .cloneMemRefs(OtherMI: Orig);
1680 break;
1681 }
1682 }
1683}
1684
1685MachineInstr &
1686ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1687 MachineBasicBlock::iterator InsertBefore,
1688 const MachineInstr &Orig) const {
1689 MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1690 MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1691 for (;;) {
1692 switch (I->getOpcode()) {
1693 case ARM::tLDRpci_pic:
1694 case ARM::t2LDRpci_pic: {
1695 MachineFunction &MF = *MBB.getParent();
1696 unsigned CPI = I->getOperand(i: 1).getIndex();
1697 unsigned PCLabelId = duplicateCPV(MF, CPI);
1698 I->getOperand(i: 1).setIndex(CPI);
1699 I->getOperand(i: 2).setImm(PCLabelId);
1700 break;
1701 }
1702 }
1703 if (!I->isBundledWithSucc())
1704 break;
1705 ++I;
1706 }
1707 return Cloned;
1708}
1709
1710bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1711 const MachineInstr &MI1,
1712 const MachineRegisterInfo *MRI) const {
1713 unsigned Opcode = MI0.getOpcode();
1714 if (Opcode == ARM::t2LDRpci || Opcode == ARM::t2LDRpci_pic ||
1715 Opcode == ARM::tLDRpci || Opcode == ARM::tLDRpci_pic ||
1716 Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1717 Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1718 Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1719 Opcode == ARM::t2MOV_ga_pcrel) {
1720 if (MI1.getOpcode() != Opcode)
1721 return false;
1722 if (MI0.getNumOperands() != MI1.getNumOperands())
1723 return false;
1724
1725 const MachineOperand &MO0 = MI0.getOperand(i: 1);
1726 const MachineOperand &MO1 = MI1.getOperand(i: 1);
1727 if (MO0.getOffset() != MO1.getOffset())
1728 return false;
1729
1730 if (Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1731 Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1732 Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1733 Opcode == ARM::t2MOV_ga_pcrel)
1734 // Ignore the PC labels.
1735 return MO0.getGlobal() == MO1.getGlobal();
1736
1737 const MachineFunction *MF = MI0.getParent()->getParent();
1738 const MachineConstantPool *MCP = MF->getConstantPool();
1739 int CPI0 = MO0.getIndex();
1740 int CPI1 = MO1.getIndex();
1741 const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1742 const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1743 bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1744 bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1745 if (isARMCP0 && isARMCP1) {
1746 ARMConstantPoolValue *ACPV0 =
1747 static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1748 ARMConstantPoolValue *ACPV1 =
1749 static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1750 return ACPV0->hasSameValue(ACPV: ACPV1);
1751 } else if (!isARMCP0 && !isARMCP1) {
1752 return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1753 }
1754 return false;
1755 } else if (Opcode == ARM::PICLDR) {
1756 if (MI1.getOpcode() != Opcode)
1757 return false;
1758 if (MI0.getNumOperands() != MI1.getNumOperands())
1759 return false;
1760
1761 Register Addr0 = MI0.getOperand(i: 1).getReg();
1762 Register Addr1 = MI1.getOperand(i: 1).getReg();
1763 if (Addr0 != Addr1) {
1764 if (!MRI || !Addr0.isVirtual() || !Addr1.isVirtual())
1765 return false;
1766
1767 // This assumes SSA form.
1768 MachineInstr *Def0 = MRI->getVRegDef(Reg: Addr0);
1769 MachineInstr *Def1 = MRI->getVRegDef(Reg: Addr1);
1770 // Check if the loaded value, e.g. a constantpool of a global address, are
1771 // the same.
1772 if (!produceSameValue(MI0: *Def0, MI1: *Def1, MRI))
1773 return false;
1774 }
1775
1776 for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1777 // %12 = PICLDR %11, 0, 14, %noreg
1778 const MachineOperand &MO0 = MI0.getOperand(i);
1779 const MachineOperand &MO1 = MI1.getOperand(i);
1780 if (!MO0.isIdenticalTo(Other: MO1))
1781 return false;
1782 }
1783 return true;
1784 }
1785
1786 return MI0.isIdenticalTo(Other: MI1, Check: MachineInstr::IgnoreVRegDefs);
1787}
1788
1789/// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1790/// determine if two loads are loading from the same base address. It should
1791/// only return true if the base pointers are the same and the only differences
1792/// between the two addresses is the offset. It also returns the offsets by
1793/// reference.
1794///
1795/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1796/// is permanently disabled.
1797bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1798 int64_t &Offset1,
1799 int64_t &Offset2) const {
1800 // Don't worry about Thumb: just ARM and Thumb2.
1801 if (Subtarget.isThumb1Only()) return false;
1802
1803 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1804 return false;
1805
1806 auto IsLoadOpcode = [&](unsigned Opcode) {
1807 switch (Opcode) {
1808 default:
1809 return false;
1810 case ARM::LDRi12:
1811 case ARM::LDRBi12:
1812 case ARM::LDRD:
1813 case ARM::LDRH:
1814 case ARM::LDRSB:
1815 case ARM::LDRSH:
1816 case ARM::VLDRD:
1817 case ARM::VLDRS:
1818 case ARM::t2LDRi8:
1819 case ARM::t2LDRBi8:
1820 case ARM::t2LDRDi8:
1821 case ARM::t2LDRSHi8:
1822 case ARM::t2LDRi12:
1823 case ARM::t2LDRBi12:
1824 case ARM::t2LDRSHi12:
1825 return true;
1826 }
1827 };
1828
1829 if (!IsLoadOpcode(Load1->getMachineOpcode()) ||
1830 !IsLoadOpcode(Load2->getMachineOpcode()))
1831 return false;
1832
1833 // Check if base addresses and chain operands match.
1834 if (Load1->getOperand(Num: 0) != Load2->getOperand(Num: 0) ||
1835 Load1->getOperand(Num: 4) != Load2->getOperand(Num: 4))
1836 return false;
1837
1838 // Index should be Reg0.
1839 if (Load1->getOperand(Num: 3) != Load2->getOperand(Num: 3))
1840 return false;
1841
1842 // Determine the offsets.
1843 if (isa<ConstantSDNode>(Val: Load1->getOperand(Num: 1)) &&
1844 isa<ConstantSDNode>(Val: Load2->getOperand(Num: 1))) {
1845 Offset1 = cast<ConstantSDNode>(Val: Load1->getOperand(Num: 1))->getSExtValue();
1846 Offset2 = cast<ConstantSDNode>(Val: Load2->getOperand(Num: 1))->getSExtValue();
1847 return true;
1848 }
1849
1850 return false;
1851}
1852
1853/// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
1854/// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
1855/// be scheduled together. On some targets if two loads are loading from
1856/// addresses in the same cache line, it's better if they are scheduled
1857/// together. This function takes two integers that represent the load offsets
1858/// from the common base address. It returns true if it decides it's desirable
1859/// to schedule the two loads together. "NumLoads" is the number of loads that
1860/// have already been scheduled after Load1.
1861///
1862/// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1863/// is permanently disabled.
1864bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
1865 int64_t Offset1, int64_t Offset2,
1866 unsigned NumLoads) const {
1867 // Don't worry about Thumb: just ARM and Thumb2.
1868 if (Subtarget.isThumb1Only()) return false;
1869
1870 assert(Offset2 > Offset1);
1871
1872 if ((Offset2 - Offset1) / 8 > 64)
1873 return false;
1874
1875 // Check if the machine opcodes are different. If they are different
1876 // then we consider them to not be of the same base address,
1877 // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
1878 // In this case, they are considered to be the same because they are different
1879 // encoding forms of the same basic instruction.
1880 if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
1881 !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
1882 Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
1883 (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
1884 Load2->getMachineOpcode() == ARM::t2LDRBi8)))
1885 return false; // FIXME: overly conservative?
1886
1887 // Four loads in a row should be sufficient.
1888 if (NumLoads >= 3)
1889 return false;
1890
1891 return true;
1892}
1893
1894bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1895 const MachineBasicBlock *MBB,
1896 const MachineFunction &MF) const {
1897 // Debug info is never a scheduling boundary. It's necessary to be explicit
1898 // due to the special treatment of IT instructions below, otherwise a
1899 // dbg_value followed by an IT will result in the IT instruction being
1900 // considered a scheduling hazard, which is wrong. It should be the actual
1901 // instruction preceding the dbg_value instruction(s), just like it is
1902 // when debug info is not present.
1903 if (MI.isDebugInstr())
1904 return false;
1905
1906 // Terminators and labels can't be scheduled around.
1907 if (MI.isTerminator() || MI.isPosition())
1908 return true;
1909
1910 // INLINEASM_BR can jump to another block
1911 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1912 return true;
1913
1914 if (isSEHInstruction(MI))
1915 return true;
1916
1917 // Treat the start of the IT block as a scheduling boundary, but schedule
1918 // t2IT along with all instructions following it.
1919 // FIXME: This is a big hammer. But the alternative is to add all potential
1920 // true and anti dependencies to IT block instructions as implicit operands
1921 // to the t2IT instruction. The added compile time and complexity does not
1922 // seem worth it.
1923 MachineBasicBlock::const_iterator I = MI;
1924 // Make sure to skip any debug instructions
1925 while (++I != MBB->end() && I->isDebugInstr())
1926 ;
1927 if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
1928 return true;
1929
1930 // Don't attempt to schedule around any instruction that defines
1931 // a stack-oriented pointer, as it's unlikely to be profitable. This
1932 // saves compile time, because it doesn't require every single
1933 // stack slot reference to depend on the instruction that does the
1934 // modification.
1935 // Calls don't actually change the stack pointer, even if they have imp-defs.
1936 // No ARM calling conventions change the stack pointer. (X86 calling
1937 // conventions sometimes do).
1938 if (!MI.isCall() && MI.definesRegister(Reg: ARM::SP, /*TRI=*/nullptr))
1939 return true;
1940
1941 return false;
1942}
1943
1944bool ARMBaseInstrInfo::
1945isProfitableToIfCvt(MachineBasicBlock &MBB,
1946 unsigned NumCycles, unsigned ExtraPredCycles,
1947 BranchProbability Probability) const {
1948 if (!NumCycles)
1949 return false;
1950
1951 // If we are optimizing for size, see if the branch in the predecessor can be
1952 // lowered to cbn?z by the constant island lowering pass, and return false if
1953 // so. This results in a shorter instruction sequence.
1954 if (MBB.getParent()->getFunction().hasOptSize()) {
1955 MachineBasicBlock *Pred = *MBB.pred_begin();
1956 if (!Pred->empty()) {
1957 MachineInstr *LastMI = &*Pred->rbegin();
1958 if (LastMI->getOpcode() == ARM::t2Bcc) {
1959 const TargetRegisterInfo *TRI = &getRegisterInfo();
1960 MachineInstr *CmpMI = findCMPToFoldIntoCBZ(Br: LastMI, TRI);
1961 if (CmpMI)
1962 return false;
1963 }
1964 }
1965 }
1966 return isProfitableToIfCvt(TMBB&: MBB, NumT: NumCycles, ExtraT: ExtraPredCycles,
1967 FMBB&: MBB, NumF: 0, ExtraF: 0, Probability);
1968}
1969
1970bool ARMBaseInstrInfo::
1971isProfitableToIfCvt(MachineBasicBlock &TBB,
1972 unsigned TCycles, unsigned TExtra,
1973 MachineBasicBlock &FBB,
1974 unsigned FCycles, unsigned FExtra,
1975 BranchProbability Probability) const {
1976 if (!TCycles)
1977 return false;
1978
1979 // In thumb code we often end up trading one branch for a IT block, and
1980 // if we are cloning the instruction can increase code size. Prevent
1981 // blocks with multiple predecessors from being ifcvted to prevent this
1982 // cloning.
1983 if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
1984 if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
1985 return false;
1986 }
1987
1988 // Attempt to estimate the relative costs of predication versus branching.
1989 // Here we scale up each component of UnpredCost to avoid precision issue when
1990 // scaling TCycles/FCycles by Probability.
1991 const unsigned ScalingUpFactor = 1024;
1992
1993 unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
1994 unsigned UnpredCost;
1995 if (!Subtarget.hasBranchPredictor()) {
1996 // When we don't have a branch predictor it's always cheaper to not take a
1997 // branch than take it, so we have to take that into account.
1998 unsigned NotTakenBranchCost = 1;
1999 unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
2000 unsigned TUnpredCycles, FUnpredCycles;
2001 if (!FCycles) {
2002 // Triangle: TBB is the fallthrough
2003 TUnpredCycles = TCycles + NotTakenBranchCost;
2004 FUnpredCycles = TakenBranchCost;
2005 } else {
2006 // Diamond: TBB is the block that is branched to, FBB is the fallthrough
2007 TUnpredCycles = TCycles + TakenBranchCost;
2008 FUnpredCycles = FCycles + NotTakenBranchCost;
2009 // The branch at the end of FBB will disappear when it's predicated, so
2010 // discount it from PredCost.
2011 PredCost -= 1 * ScalingUpFactor;
2012 }
2013 // The total cost is the cost of each path scaled by their probabilities
2014 unsigned TUnpredCost = Probability.scale(Num: TUnpredCycles * ScalingUpFactor);
2015 unsigned FUnpredCost = Probability.getCompl().scale(Num: FUnpredCycles * ScalingUpFactor);
2016 UnpredCost = TUnpredCost + FUnpredCost;
2017 // When predicating assume that the first IT can be folded away but later
2018 // ones cost one cycle each
2019 if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2020 PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2021 }
2022 } else {
2023 unsigned TUnpredCost = Probability.scale(Num: TCycles * ScalingUpFactor);
2024 unsigned FUnpredCost =
2025 Probability.getCompl().scale(Num: FCycles * ScalingUpFactor);
2026 UnpredCost = TUnpredCost + FUnpredCost;
2027 UnpredCost += 1 * ScalingUpFactor; // The branch itself
2028 UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2029 }
2030
2031 return PredCost <= UnpredCost;
2032}
2033
2034unsigned
2035ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2036 unsigned NumInsts) const {
2037 // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2038 // ARM has a condition code field in every predicable instruction, using it
2039 // doesn't change code size.
2040 if (!Subtarget.isThumb2())
2041 return 0;
2042
2043 // It's possible that the size of the IT is restricted to a single block.
2044 unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4;
2045 return divideCeil(Numerator: NumInsts, Denominator: MaxInsts) * 2;
2046}
2047
2048unsigned
2049ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2050 // If this branch is likely to be folded into the comparison to form a
2051 // CB(N)Z, then removing it won't reduce code size at all, because that will
2052 // just replace the CB(N)Z with a CMP.
2053 if (MI.getOpcode() == ARM::t2Bcc &&
2054 findCMPToFoldIntoCBZ(Br: &MI, TRI: &getRegisterInfo()))
2055 return 0;
2056
2057 unsigned Size = getInstSizeInBytes(MI);
2058
2059 // For Thumb2, all branches are 32-bit instructions during the if conversion
2060 // pass, but may be replaced with 16-bit instructions during size reduction.
2061 // Since the branches considered by if conversion tend to be forward branches
2062 // over small basic blocks, they are very likely to be in range for the
2063 // narrow instructions, so we assume the final code size will be half what it
2064 // currently is.
2065 if (Subtarget.isThumb2())
2066 Size /= 2;
2067
2068 return Size;
2069}
2070
2071bool
2072ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2073 MachineBasicBlock &FMBB) const {
2074 // Reduce false anti-dependencies to let the target's out-of-order execution
2075 // engine do its thing.
2076 return Subtarget.isProfitableToUnpredicate();
2077}
2078
2079/// getInstrPredicate - If instruction is predicated, returns its predicate
2080/// condition, otherwise returns AL. It also returns the condition code
2081/// register by reference.
2082ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2083 Register &PredReg) {
2084 int PIdx = MI.findFirstPredOperandIdx();
2085 if (PIdx == -1) {
2086 PredReg = 0;
2087 return ARMCC::AL;
2088 }
2089
2090 PredReg = MI.getOperand(i: PIdx+1).getReg();
2091 return (ARMCC::CondCodes)MI.getOperand(i: PIdx).getImm();
2092}
2093
2094unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2095 if (Opc == ARM::B)
2096 return ARM::Bcc;
2097 if (Opc == ARM::tB)
2098 return ARM::tBcc;
2099 if (Opc == ARM::t2B)
2100 return ARM::t2Bcc;
2101
2102 llvm_unreachable("Unknown unconditional branch opcode!");
2103}
2104
2105MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2106 bool NewMI,
2107 unsigned OpIdx1,
2108 unsigned OpIdx2) const {
2109 switch (MI.getOpcode()) {
2110 case ARM::MOVCCr:
2111 case ARM::t2MOVCCr: {
2112 // MOVCC can be commuted by inverting the condition.
2113 Register PredReg;
2114 ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2115 // MOVCC AL can't be inverted. Shouldn't happen.
2116 if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2117 return nullptr;
2118 MachineInstr *CommutedMI =
2119 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2120 if (!CommutedMI)
2121 return nullptr;
2122 // After swapping the MOVCC operands, also invert the condition.
2123 CommutedMI->getOperand(i: CommutedMI->findFirstPredOperandIdx())
2124 .setImm(ARMCC::getOppositeCondition(CC));
2125 return CommutedMI;
2126 }
2127 }
2128 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2129}
2130
2131/// Identify instructions that can be folded into a MOVCC instruction, and
2132/// return the defining instruction.
2133MachineInstr *
2134ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2135 const TargetInstrInfo *TII) const {
2136 if (!Reg.isVirtual())
2137 return nullptr;
2138 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
2139 return nullptr;
2140 MachineInstr *MI = MRI.getVRegDef(Reg);
2141 if (!MI)
2142 return nullptr;
2143 // Check if MI can be predicated and folded into the MOVCC.
2144 if (!isPredicable(MI: *MI))
2145 return nullptr;
2146 // Check if MI has any non-dead defs or physreg uses. This also detects
2147 // predicated instructions which will be reading CPSR.
2148 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 1)) {
2149 // Reject frame index operands, PEI can't handle the predicated pseudos.
2150 if (MO.isFI() || MO.isCPI() || MO.isJTI())
2151 return nullptr;
2152 if (!MO.isReg())
2153 continue;
2154 // MI can't have any tied operands, that would conflict with predication.
2155 if (MO.isTied())
2156 return nullptr;
2157 if (MO.getReg().isPhysical())
2158 return nullptr;
2159 if (MO.isDef() && !MO.isDead())
2160 return nullptr;
2161 }
2162 bool DontMoveAcrossStores = true;
2163 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
2164 return nullptr;
2165 return MI;
2166}
2167
2168MachineInstr *
2169ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2170 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2171 bool PreferFalse) const {
2172 assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2173 "Unknown select instruction");
2174 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2175 MachineInstr *DefMI = canFoldIntoMOVCC(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this);
2176 bool Invert = !DefMI;
2177 if (!DefMI)
2178 DefMI = canFoldIntoMOVCC(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this);
2179 if (!DefMI)
2180 return nullptr;
2181
2182 // Find new register class to use.
2183 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
2184 MachineOperand TrueReg = MI.getOperand(i: Invert ? 1 : 2);
2185 Register DestReg = MI.getOperand(i: 0).getReg();
2186 const TargetRegisterClass *FalseClass = MRI.getRegClass(Reg: FalseReg.getReg());
2187 const TargetRegisterClass *TrueClass = MRI.getRegClass(Reg: TrueReg.getReg());
2188 if (!MRI.constrainRegClass(Reg: DestReg, RC: FalseClass))
2189 return nullptr;
2190 if (!MRI.constrainRegClass(Reg: DestReg, RC: TrueClass))
2191 return nullptr;
2192
2193 // Create a new predicated version of DefMI.
2194 // Rfalse is the first use.
2195 MachineInstrBuilder NewMI =
2196 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: DefMI->getDesc(), DestReg);
2197
2198 // Copy all the DefMI operands, excluding its (null) predicate.
2199 const MCInstrDesc &DefDesc = DefMI->getDesc();
2200 for (unsigned i = 1, e = DefDesc.getNumOperands();
2201 i != e && !DefDesc.operands()[i].isPredicate(); ++i)
2202 NewMI.add(MO: DefMI->getOperand(i));
2203
2204 unsigned CondCode = MI.getOperand(i: 3).getImm();
2205 if (Invert)
2206 NewMI.addImm(Val: ARMCC::getOppositeCondition(CC: ARMCC::CondCodes(CondCode)));
2207 else
2208 NewMI.addImm(Val: CondCode);
2209 NewMI.add(MO: MI.getOperand(i: 4));
2210
2211 // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2212 if (NewMI->hasOptionalDef())
2213 NewMI.add(MO: condCodeOp());
2214
2215 // The output register value when the predicate is false is an implicit
2216 // register operand tied to the first def.
2217 // The tie makes the register allocator ensure the FalseReg is allocated the
2218 // same register as operand 0.
2219 FalseReg.setImplicit();
2220 NewMI.add(MO: FalseReg);
2221 NewMI->tieOperands(DefIdx: 0, UseIdx: NewMI->getNumOperands() - 1);
2222
2223 // Update SeenMIs set: register newly created MI and erase removed DefMI.
2224 SeenMIs.insert(Ptr: NewMI);
2225 SeenMIs.erase(Ptr: DefMI);
2226
2227 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2228 // DefMI would be invalid when transferred inside the loop. Checking for a
2229 // loop is expensive, but at least remove kill flags if they are in different
2230 // BBs.
2231 if (DefMI->getParent() != MI.getParent())
2232 NewMI->clearKillInfo();
2233
2234 // The caller will erase MI, but not DefMI.
2235 DefMI->eraseFromParent();
2236 return NewMI;
2237}
2238
2239/// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2240/// instruction is encoded with an 'S' bit is determined by the optional CPSR
2241/// def operand.
2242///
2243/// This will go away once we can teach tblgen how to set the optional CPSR def
2244/// operand itself.
2245struct AddSubFlagsOpcodePair {
2246 uint16_t PseudoOpc;
2247 uint16_t MachineOpc;
2248};
2249
2250static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2251 {.PseudoOpc: ARM::ADDSri, .MachineOpc: ARM::ADDri},
2252 {.PseudoOpc: ARM::ADDSrr, .MachineOpc: ARM::ADDrr},
2253 {.PseudoOpc: ARM::ADDSrsi, .MachineOpc: ARM::ADDrsi},
2254 {.PseudoOpc: ARM::ADDSrsr, .MachineOpc: ARM::ADDrsr},
2255
2256 {.PseudoOpc: ARM::SUBSri, .MachineOpc: ARM::SUBri},
2257 {.PseudoOpc: ARM::SUBSrr, .MachineOpc: ARM::SUBrr},
2258 {.PseudoOpc: ARM::SUBSrsi, .MachineOpc: ARM::SUBrsi},
2259 {.PseudoOpc: ARM::SUBSrsr, .MachineOpc: ARM::SUBrsr},
2260
2261 {.PseudoOpc: ARM::RSBSri, .MachineOpc: ARM::RSBri},
2262 {.PseudoOpc: ARM::RSBSrsi, .MachineOpc: ARM::RSBrsi},
2263 {.PseudoOpc: ARM::RSBSrsr, .MachineOpc: ARM::RSBrsr},
2264
2265 {.PseudoOpc: ARM::tADDSi3, .MachineOpc: ARM::tADDi3},
2266 {.PseudoOpc: ARM::tADDSi8, .MachineOpc: ARM::tADDi8},
2267 {.PseudoOpc: ARM::tADDSrr, .MachineOpc: ARM::tADDrr},
2268 {.PseudoOpc: ARM::tADCS, .MachineOpc: ARM::tADC},
2269
2270 {.PseudoOpc: ARM::tSUBSi3, .MachineOpc: ARM::tSUBi3},
2271 {.PseudoOpc: ARM::tSUBSi8, .MachineOpc: ARM::tSUBi8},
2272 {.PseudoOpc: ARM::tSUBSrr, .MachineOpc: ARM::tSUBrr},
2273 {.PseudoOpc: ARM::tSBCS, .MachineOpc: ARM::tSBC},
2274 {.PseudoOpc: ARM::tRSBS, .MachineOpc: ARM::tRSB},
2275 {.PseudoOpc: ARM::tLSLSri, .MachineOpc: ARM::tLSLri},
2276
2277 {.PseudoOpc: ARM::t2ADDSri, .MachineOpc: ARM::t2ADDri},
2278 {.PseudoOpc: ARM::t2ADDSrr, .MachineOpc: ARM::t2ADDrr},
2279 {.PseudoOpc: ARM::t2ADDSrs, .MachineOpc: ARM::t2ADDrs},
2280
2281 {.PseudoOpc: ARM::t2SUBSri, .MachineOpc: ARM::t2SUBri},
2282 {.PseudoOpc: ARM::t2SUBSrr, .MachineOpc: ARM::t2SUBrr},
2283 {.PseudoOpc: ARM::t2SUBSrs, .MachineOpc: ARM::t2SUBrs},
2284
2285 {.PseudoOpc: ARM::t2RSBSri, .MachineOpc: ARM::t2RSBri},
2286 {.PseudoOpc: ARM::t2RSBSrs, .MachineOpc: ARM::t2RSBrs},
2287};
2288
2289unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2290 for (const auto &Entry : AddSubFlagsOpcodeMap)
2291 if (OldOpc == Entry.PseudoOpc)
2292 return Entry.MachineOpc;
2293 return 0;
2294}
2295
2296void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2297 MachineBasicBlock::iterator &MBBI,
2298 const DebugLoc &dl, Register DestReg,
2299 Register BaseReg, int NumBytes,
2300 ARMCC::CondCodes Pred, Register PredReg,
2301 const ARMBaseInstrInfo &TII,
2302 unsigned MIFlags) {
2303 if (NumBytes == 0 && DestReg != BaseReg) {
2304 BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: ARM::MOVr), DestReg)
2305 .addReg(RegNo: BaseReg, Flags: RegState::Kill)
2306 .add(MOs: predOps(Pred, PredReg))
2307 .add(MO: condCodeOp())
2308 .setMIFlags(MIFlags);
2309 return;
2310 }
2311
2312 bool isSub = NumBytes < 0;
2313 if (isSub) NumBytes = -NumBytes;
2314
2315 while (NumBytes) {
2316 unsigned RotAmt = ARM_AM::getSOImmValRotate(Imm: NumBytes);
2317 unsigned ThisVal = NumBytes & llvm::rotr<uint32_t>(V: 0xFF, R: RotAmt);
2318 assert(ThisVal && "Didn't extract field correctly");
2319
2320 // We will handle these bits from offset, clear them.
2321 NumBytes &= ~ThisVal;
2322
2323 assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2324
2325 // Build the new ADD / SUB.
2326 unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2327 BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: Opc), DestReg)
2328 .addReg(RegNo: BaseReg, Flags: RegState::Kill)
2329 .addImm(Val: ThisVal)
2330 .add(MOs: predOps(Pred, PredReg))
2331 .add(MO: condCodeOp())
2332 .setMIFlags(MIFlags);
2333 BaseReg = DestReg;
2334 }
2335}
2336
2337bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2338 MachineFunction &MF, MachineInstr *MI,
2339 unsigned NumBytes) {
2340 // This optimisation potentially adds lots of load and store
2341 // micro-operations, it's only really a great benefit to code-size.
2342 if (!Subtarget.hasMinSize())
2343 return false;
2344
2345 // If only one register is pushed/popped, LLVM can use an LDR/STR
2346 // instead. We can't modify those so make sure we're dealing with an
2347 // instruction we understand.
2348 bool IsPop = isPopOpcode(Opc: MI->getOpcode());
2349 bool IsPush = isPushOpcode(Opc: MI->getOpcode());
2350 if (!IsPush && !IsPop)
2351 return false;
2352
2353 bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2354 MI->getOpcode() == ARM::VLDMDIA_UPD;
2355 bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2356 MI->getOpcode() == ARM::tPOP ||
2357 MI->getOpcode() == ARM::tPOP_RET;
2358
2359 assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2360 MI->getOperand(1).getReg() == ARM::SP)) &&
2361 "trying to fold sp update into non-sp-updating push/pop");
2362
2363 // The VFP push & pop act on D-registers, so we can only fold an adjustment
2364 // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2365 // if this is violated.
2366 if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2367 return false;
2368
2369 // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2370 // pred) so the list starts at 4. Thumb1 starts after the predicate.
2371 int RegListIdx = IsT1PushPop ? 2 : 4;
2372
2373 // Calculate the space we'll need in terms of registers.
2374 unsigned RegsNeeded;
2375 const TargetRegisterClass *RegClass;
2376 if (IsVFPPushPop) {
2377 RegsNeeded = NumBytes / 8;
2378 RegClass = &ARM::DPRRegClass;
2379 } else {
2380 RegsNeeded = NumBytes / 4;
2381 RegClass = &ARM::GPRRegClass;
2382 }
2383
2384 // We're going to have to strip all list operands off before
2385 // re-adding them since the order matters, so save the existing ones
2386 // for later.
2387 SmallVector<MachineOperand, 4> RegList;
2388
2389 // We're also going to need the first register transferred by this
2390 // instruction, which won't necessarily be the first register in the list.
2391 unsigned FirstRegEnc = -1;
2392
2393 const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2394 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2395 MachineOperand &MO = MI->getOperand(i);
2396 RegList.push_back(Elt: MO);
2397
2398 if (MO.isReg() && !MO.isImplicit() &&
2399 TRI->getEncodingValue(Reg: MO.getReg()) < FirstRegEnc)
2400 FirstRegEnc = TRI->getEncodingValue(Reg: MO.getReg());
2401 }
2402
2403 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(MF: &MF);
2404
2405 // Now try to find enough space in the reglist to allocate NumBytes.
2406 for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2407 --CurRegEnc) {
2408 MCRegister CurReg = RegClass->getRegister(i: CurRegEnc);
2409 if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(Reg: ARM::R7))
2410 continue;
2411 if (!IsPop) {
2412 // Pushing any register is completely harmless, mark the register involved
2413 // as undef since we don't care about its value and must not restore it
2414 // during stack unwinding.
2415 RegList.push_back(Elt: MachineOperand::CreateReg(Reg: CurReg, isDef: false, isImp: false,
2416 isKill: false, isDead: false, isUndef: true));
2417 --RegsNeeded;
2418 continue;
2419 }
2420
2421 // However, we can only pop an extra register if it's not live. For
2422 // registers live within the function we might clobber a return value
2423 // register; the other way a register can be live here is if it's
2424 // callee-saved.
2425 if (isCalleeSavedRegister(Reg: CurReg, CSRegs) ||
2426 MI->getParent()->computeRegisterLiveness(TRI, Reg: CurReg, Before: MI) !=
2427 MachineBasicBlock::LQR_Dead) {
2428 // VFP pops don't allow holes in the register list, so any skip is fatal
2429 // for our transformation. GPR pops do, so we should just keep looking.
2430 if (IsVFPPushPop)
2431 return false;
2432 else
2433 continue;
2434 }
2435
2436 // Mark the unimportant registers as <def,dead> in the POP.
2437 RegList.push_back(Elt: MachineOperand::CreateReg(Reg: CurReg, isDef: true, isImp: false, isKill: false,
2438 isDead: true));
2439 --RegsNeeded;
2440 }
2441
2442 if (RegsNeeded > 0)
2443 return false;
2444
2445 // Finally we know we can profitably perform the optimisation so go
2446 // ahead: strip all existing registers off and add them back again
2447 // in the right order.
2448 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2449 MI->removeOperand(OpNo: i);
2450
2451 // Add the complete list back in.
2452 MachineInstrBuilder MIB(MF, &*MI);
2453 for (const MachineOperand &MO : llvm::reverse(C&: RegList))
2454 MIB.add(MO);
2455
2456 return true;
2457}
2458
2459bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2460 Register FrameReg, int &Offset,
2461 const ARMBaseInstrInfo &TII) {
2462 unsigned Opcode = MI.getOpcode();
2463 const MCInstrDesc &Desc = MI.getDesc();
2464 unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2465 bool isSub = false;
2466
2467 // Memory operands in inline assembly always use AddrMode2.
2468 if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2469 AddrMode = ARMII::AddrMode2;
2470
2471 if (Opcode == ARM::ADDri) {
2472 Offset += MI.getOperand(i: FrameRegIdx+1).getImm();
2473 if (Offset == 0) {
2474 // Turn it into a move.
2475 MI.setDesc(TII.get(Opcode: ARM::MOVr));
2476 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2477 MI.removeOperand(OpNo: FrameRegIdx+1);
2478 Offset = 0;
2479 return true;
2480 } else if (Offset < 0) {
2481 Offset = -Offset;
2482 isSub = true;
2483 MI.setDesc(TII.get(Opcode: ARM::SUBri));
2484 }
2485
2486 // Common case: small offset, fits into instruction.
2487 if (ARM_AM::getSOImmVal(Arg: Offset) != -1) {
2488 // Replace the FrameIndex with sp / fp
2489 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2490 MI.getOperand(i: FrameRegIdx+1).ChangeToImmediate(ImmVal: Offset);
2491 Offset = 0;
2492 return true;
2493 }
2494
2495 // Otherwise, pull as much of the immediate into this ADDri/SUBri
2496 // as possible.
2497 unsigned RotAmt = ARM_AM::getSOImmValRotate(Imm: Offset);
2498 unsigned ThisImmVal = Offset & llvm::rotr<uint32_t>(V: 0xFF, R: RotAmt);
2499
2500 // We will handle these bits from offset, clear them.
2501 Offset &= ~ThisImmVal;
2502
2503 // Get the properly encoded SOImmVal field.
2504 assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2505 "Bit extraction didn't work?");
2506 MI.getOperand(i: FrameRegIdx+1).ChangeToImmediate(ImmVal: ThisImmVal);
2507 } else {
2508 unsigned ImmIdx = 0;
2509 int InstrOffs = 0;
2510 unsigned NumBits = 0;
2511 unsigned Scale = 1;
2512 switch (AddrMode) {
2513 case ARMII::AddrMode_i12:
2514 ImmIdx = FrameRegIdx + 1;
2515 InstrOffs = MI.getOperand(i: ImmIdx).getImm();
2516 NumBits = 12;
2517 break;
2518 case ARMII::AddrMode2:
2519 ImmIdx = FrameRegIdx+2;
2520 InstrOffs = ARM_AM::getAM2Offset(AM2Opc: MI.getOperand(i: ImmIdx).getImm());
2521 if (ARM_AM::getAM2Op(AM2Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2522 InstrOffs *= -1;
2523 NumBits = 12;
2524 break;
2525 case ARMII::AddrMode3:
2526 ImmIdx = FrameRegIdx+2;
2527 InstrOffs = ARM_AM::getAM3Offset(AM3Opc: MI.getOperand(i: ImmIdx).getImm());
2528 if (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2529 InstrOffs *= -1;
2530 NumBits = 8;
2531 break;
2532 case ARMII::AddrMode4:
2533 case ARMII::AddrMode6:
2534 // Can't fold any offset even if it's zero.
2535 return false;
2536 case ARMII::AddrMode5:
2537 ImmIdx = FrameRegIdx+1;
2538 InstrOffs = ARM_AM::getAM5Offset(AM5Opc: MI.getOperand(i: ImmIdx).getImm());
2539 if (ARM_AM::getAM5Op(AM5Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2540 InstrOffs *= -1;
2541 NumBits = 8;
2542 Scale = 4;
2543 break;
2544 case ARMII::AddrMode5FP16:
2545 ImmIdx = FrameRegIdx+1;
2546 InstrOffs = ARM_AM::getAM5Offset(AM5Opc: MI.getOperand(i: ImmIdx).getImm());
2547 if (ARM_AM::getAM5Op(AM5Opc: MI.getOperand(i: ImmIdx).getImm()) == ARM_AM::sub)
2548 InstrOffs *= -1;
2549 NumBits = 8;
2550 Scale = 2;
2551 break;
2552 case ARMII::AddrModeT2_i7:
2553 case ARMII::AddrModeT2_i7s2:
2554 case ARMII::AddrModeT2_i7s4:
2555 ImmIdx = FrameRegIdx+1;
2556 InstrOffs = MI.getOperand(i: ImmIdx).getImm();
2557 NumBits = 7;
2558 Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2559 AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2560 break;
2561 default:
2562 llvm_unreachable("Unsupported addressing mode!");
2563 }
2564
2565 Offset += InstrOffs * Scale;
2566 assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2567 if (Offset < 0) {
2568 Offset = -Offset;
2569 isSub = true;
2570 }
2571
2572 // Attempt to fold address comp. if opcode has offset bits
2573 if (NumBits > 0) {
2574 // Common case: small offset, fits into instruction.
2575 MachineOperand &ImmOp = MI.getOperand(i: ImmIdx);
2576 int ImmedOffset = Offset / Scale;
2577 unsigned Mask = (1 << NumBits) - 1;
2578 if ((unsigned)Offset <= Mask * Scale) {
2579 // Replace the FrameIndex with sp
2580 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
2581 // FIXME: When addrmode2 goes away, this will simplify (like the
2582 // T2 version), as the LDR.i12 versions don't need the encoding
2583 // tricks for the offset value.
2584 if (isSub) {
2585 if (AddrMode == ARMII::AddrMode_i12)
2586 ImmedOffset = -ImmedOffset;
2587 else
2588 ImmedOffset |= 1 << NumBits;
2589 }
2590 ImmOp.ChangeToImmediate(ImmVal: ImmedOffset);
2591 Offset = 0;
2592 return true;
2593 }
2594
2595 // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2596 ImmedOffset = ImmedOffset & Mask;
2597 if (isSub) {
2598 if (AddrMode == ARMII::AddrMode_i12)
2599 ImmedOffset = -ImmedOffset;
2600 else
2601 ImmedOffset |= 1 << NumBits;
2602 }
2603 ImmOp.ChangeToImmediate(ImmVal: ImmedOffset);
2604 Offset &= ~(Mask*Scale);
2605 }
2606 }
2607
2608 Offset = (isSub) ? -Offset : Offset;
2609 return Offset == 0;
2610}
2611
2612/// analyzeCompare - For a comparison instruction, return the source registers
2613/// in SrcReg and SrcReg2 if having two register operands, and the value it
2614/// compares against in CmpValue. Return true if the comparison instruction
2615/// can be analyzed.
2616bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2617 Register &SrcReg2, int64_t &CmpMask,
2618 int64_t &CmpValue) const {
2619 switch (MI.getOpcode()) {
2620 default: break;
2621 case ARM::CMPri:
2622 case ARM::t2CMPri:
2623 case ARM::tCMPi8:
2624 SrcReg = MI.getOperand(i: 0).getReg();
2625 SrcReg2 = 0;
2626 CmpMask = ~0;
2627 CmpValue = MI.getOperand(i: 1).getImm();
2628 return true;
2629 case ARM::CMPrr:
2630 case ARM::t2CMPrr:
2631 case ARM::tCMPr:
2632 SrcReg = MI.getOperand(i: 0).getReg();
2633 SrcReg2 = MI.getOperand(i: 1).getReg();
2634 CmpMask = ~0;
2635 CmpValue = 0;
2636 return true;
2637 case ARM::TSTri:
2638 case ARM::t2TSTri:
2639 SrcReg = MI.getOperand(i: 0).getReg();
2640 SrcReg2 = 0;
2641 CmpMask = MI.getOperand(i: 1).getImm();
2642 CmpValue = 0;
2643 return true;
2644 }
2645
2646 return false;
2647}
2648
2649/// isSuitableForMask - Identify a suitable 'and' instruction that
2650/// operates on the given source register and applies the same mask
2651/// as a 'tst' instruction. Provide a limited look-through for copies.
2652/// When successful, MI will hold the found instruction.
2653static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2654 int CmpMask, bool CommonUse) {
2655 switch (MI->getOpcode()) {
2656 case ARM::ANDri:
2657 case ARM::t2ANDri:
2658 if (CmpMask != MI->getOperand(i: 2).getImm())
2659 return false;
2660 if (SrcReg == MI->getOperand(i: CommonUse ? 1 : 0).getReg())
2661 return true;
2662 break;
2663 }
2664
2665 return false;
2666}
2667
2668/// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2669/// the condition code if we modify the instructions such that flags are
2670/// set by ADD(a,b,X).
2671inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2672 switch (CC) {
2673 default: return ARMCC::AL;
2674 case ARMCC::HS: return ARMCC::LO;
2675 case ARMCC::LO: return ARMCC::HS;
2676 case ARMCC::VS: return ARMCC::VS;
2677 case ARMCC::VC: return ARMCC::VC;
2678 }
2679}
2680
2681/// isRedundantFlagInstr - check whether the first instruction, whose only
2682/// purpose is to update flags, can be made redundant.
2683/// CMPrr can be made redundant by SUBrr if the operands are the same.
2684/// CMPri can be made redundant by SUBri if the operands are the same.
2685/// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2686/// This function can be extended later on.
2687inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2688 Register SrcReg, Register SrcReg2,
2689 int64_t ImmValue,
2690 const MachineInstr *OI,
2691 bool &IsThumb1) {
2692 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2693 (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2694 ((OI->getOperand(i: 1).getReg() == SrcReg &&
2695 OI->getOperand(i: 2).getReg() == SrcReg2) ||
2696 (OI->getOperand(i: 1).getReg() == SrcReg2 &&
2697 OI->getOperand(i: 2).getReg() == SrcReg))) {
2698 IsThumb1 = false;
2699 return true;
2700 }
2701
2702 if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2703 ((OI->getOperand(i: 2).getReg() == SrcReg &&
2704 OI->getOperand(i: 3).getReg() == SrcReg2) ||
2705 (OI->getOperand(i: 2).getReg() == SrcReg2 &&
2706 OI->getOperand(i: 3).getReg() == SrcReg))) {
2707 IsThumb1 = true;
2708 return true;
2709 }
2710
2711 if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2712 (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2713 OI->getOperand(i: 1).getReg() == SrcReg &&
2714 OI->getOperand(i: 2).getImm() == ImmValue) {
2715 IsThumb1 = false;
2716 return true;
2717 }
2718
2719 if (CmpI->getOpcode() == ARM::tCMPi8 &&
2720 (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2721 OI->getOperand(i: 2).getReg() == SrcReg &&
2722 OI->getOperand(i: 3).getImm() == ImmValue) {
2723 IsThumb1 = true;
2724 return true;
2725 }
2726
2727 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2728 (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2729 OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2730 OI->getOperand(i: 0).isReg() && OI->getOperand(i: 1).isReg() &&
2731 OI->getOperand(i: 0).getReg() == SrcReg &&
2732 OI->getOperand(i: 1).getReg() == SrcReg2) {
2733 IsThumb1 = false;
2734 return true;
2735 }
2736
2737 if (CmpI->getOpcode() == ARM::tCMPr &&
2738 (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2739 OI->getOpcode() == ARM::tADDrr) &&
2740 OI->getOperand(i: 0).getReg() == SrcReg &&
2741 OI->getOperand(i: 2).getReg() == SrcReg2) {
2742 IsThumb1 = true;
2743 return true;
2744 }
2745
2746 return false;
2747}
2748
2749static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2750 switch (MI->getOpcode()) {
2751 default: return false;
2752 case ARM::tLSLri:
2753 case ARM::tLSRri:
2754 case ARM::tLSLrr:
2755 case ARM::tLSRrr:
2756 case ARM::tSUBrr:
2757 case ARM::tADDrr:
2758 case ARM::tADDi3:
2759 case ARM::tADDi8:
2760 case ARM::tSUBi3:
2761 case ARM::tSUBi8:
2762 case ARM::tMUL:
2763 case ARM::tADC:
2764 case ARM::tSBC:
2765 case ARM::tRSB:
2766 case ARM::tAND:
2767 case ARM::tORR:
2768 case ARM::tEOR:
2769 case ARM::tBIC:
2770 case ARM::tMVN:
2771 case ARM::tASRri:
2772 case ARM::tASRrr:
2773 case ARM::tROR:
2774 IsThumb1 = true;
2775 [[fallthrough]];
2776 case ARM::RSBrr:
2777 case ARM::RSBri:
2778 case ARM::RSCrr:
2779 case ARM::RSCri:
2780 case ARM::ADDrr:
2781 case ARM::ADDri:
2782 case ARM::ADCrr:
2783 case ARM::ADCri:
2784 case ARM::SUBrr:
2785 case ARM::SUBri:
2786 case ARM::SBCrr:
2787 case ARM::SBCri:
2788 case ARM::t2RSBri:
2789 case ARM::t2ADDrr:
2790 case ARM::t2ADDri:
2791 case ARM::t2ADCrr:
2792 case ARM::t2ADCri:
2793 case ARM::t2SUBrr:
2794 case ARM::t2SUBri:
2795 case ARM::t2SBCrr:
2796 case ARM::t2SBCri:
2797 case ARM::ANDrr:
2798 case ARM::ANDri:
2799 case ARM::ANDrsr:
2800 case ARM::ANDrsi:
2801 case ARM::t2ANDrr:
2802 case ARM::t2ANDri:
2803 case ARM::t2ANDrs:
2804 case ARM::ORRrr:
2805 case ARM::ORRri:
2806 case ARM::ORRrsr:
2807 case ARM::ORRrsi:
2808 case ARM::t2ORRrr:
2809 case ARM::t2ORRri:
2810 case ARM::t2ORRrs:
2811 case ARM::EORrr:
2812 case ARM::EORri:
2813 case ARM::EORrsr:
2814 case ARM::EORrsi:
2815 case ARM::t2EORrr:
2816 case ARM::t2EORri:
2817 case ARM::t2EORrs:
2818 case ARM::BICri:
2819 case ARM::BICrr:
2820 case ARM::BICrsi:
2821 case ARM::BICrsr:
2822 case ARM::t2BICri:
2823 case ARM::t2BICrr:
2824 case ARM::t2BICrs:
2825 case ARM::t2LSRri:
2826 case ARM::t2LSRrr:
2827 case ARM::t2LSLri:
2828 case ARM::t2LSLrr:
2829 case ARM::MOVsr:
2830 case ARM::MOVsi:
2831 return true;
2832 }
2833}
2834
2835/// optimizeCompareInstr - Convert the instruction supplying the argument to the
2836/// comparison into one that sets the zero bit in the flags register;
2837/// Remove a redundant Compare instruction if an earlier instruction can set the
2838/// flags in the same way as Compare.
2839/// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
2840/// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
2841/// condition code of instructions which use the flags.
2842bool ARMBaseInstrInfo::optimizeCompareInstr(
2843 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2844 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2845 // Get the unique definition of SrcReg.
2846 MachineInstr *MI = MRI->getUniqueVRegDef(Reg: SrcReg);
2847 if (!MI) return false;
2848
2849 // Masked compares sometimes use the same register as the corresponding 'and'.
2850 if (CmpMask != ~0) {
2851 if (!isSuitableForMask(MI, SrcReg, CmpMask, CommonUse: false) || isPredicated(MI: *MI)) {
2852 MI = nullptr;
2853 for (MachineRegisterInfo::use_instr_iterator
2854 UI = MRI->use_instr_begin(RegNo: SrcReg), UE = MRI->use_instr_end();
2855 UI != UE; ++UI) {
2856 if (UI->getParent() != CmpInstr.getParent())
2857 continue;
2858 MachineInstr *PotentialAND = &*UI;
2859 if (!isSuitableForMask(MI&: PotentialAND, SrcReg, CmpMask, CommonUse: true) ||
2860 isPredicated(MI: *PotentialAND))
2861 continue;
2862 MI = PotentialAND;
2863 break;
2864 }
2865 if (!MI) return false;
2866 }
2867 }
2868
2869 // Get ready to iterate backward from CmpInstr.
2870 MachineBasicBlock::iterator I = CmpInstr, E = MI,
2871 B = CmpInstr.getParent()->begin();
2872
2873 // Early exit if CmpInstr is at the beginning of the BB.
2874 if (I == B) return false;
2875
2876 // There are two possible candidates which can be changed to set CPSR:
2877 // One is MI, the other is a SUB or ADD instruction.
2878 // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
2879 // ADDr[ri](r1, r2, X).
2880 // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
2881 MachineInstr *SubAdd = nullptr;
2882 if (SrcReg2 != 0)
2883 // MI is not a candidate for CMPrr.
2884 MI = nullptr;
2885 else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
2886 // Conservatively refuse to convert an instruction which isn't in the same
2887 // BB as the comparison.
2888 // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
2889 // Thus we cannot return here.
2890 if (CmpInstr.getOpcode() == ARM::CMPri ||
2891 CmpInstr.getOpcode() == ARM::t2CMPri ||
2892 CmpInstr.getOpcode() == ARM::tCMPi8)
2893 MI = nullptr;
2894 else
2895 return false;
2896 }
2897
2898 bool IsThumb1 = false;
2899 if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
2900 return false;
2901
2902 // We also want to do this peephole for cases like this: if (a*b == 0),
2903 // and optimise away the CMP instruction from the generated code sequence:
2904 // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
2905 // resulting from the select instruction, but these MOVS instructions for
2906 // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
2907 // However, if we only have MOVS instructions in between the CMP and the
2908 // other instruction (the MULS in this example), then the CPSR is dead so we
2909 // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
2910 // reordering and then continue the analysis hoping we can eliminate the
2911 // CMP. This peephole works on the vregs, so is still in SSA form. As a
2912 // consequence, the movs won't redefine/kill the MUL operands which would
2913 // make this reordering illegal.
2914 const TargetRegisterInfo *TRI = &getRegisterInfo();
2915 if (MI && IsThumb1) {
2916 --I;
2917 if (I != E && !MI->readsRegister(Reg: ARM::CPSR, TRI)) {
2918 bool CanReorder = true;
2919 for (; I != E; --I) {
2920 if (I->getOpcode() != ARM::tMOVi8) {
2921 CanReorder = false;
2922 break;
2923 }
2924 }
2925 if (CanReorder) {
2926 MI = MI->removeFromParent();
2927 E = CmpInstr;
2928 CmpInstr.getParent()->insert(I: E, MI);
2929 }
2930 }
2931 I = CmpInstr;
2932 E = MI;
2933 }
2934
2935 // Check that CPSR isn't set between the comparison instruction and the one we
2936 // want to change. At the same time, search for SubAdd.
2937 bool SubAddIsThumb1 = false;
2938 do {
2939 const MachineInstr &Instr = *--I;
2940
2941 // Check whether CmpInstr can be made redundant by the current instruction.
2942 if (isRedundantFlagInstr(CmpI: &CmpInstr, SrcReg, SrcReg2, ImmValue: CmpValue, OI: &Instr,
2943 IsThumb1&: SubAddIsThumb1)) {
2944 SubAdd = &*I;
2945 break;
2946 }
2947
2948 // Allow E (which was initially MI) to be SubAdd but do not search before E.
2949 if (I == E)
2950 break;
2951
2952 if (Instr.modifiesRegister(Reg: ARM::CPSR, TRI) ||
2953 Instr.readsRegister(Reg: ARM::CPSR, TRI))
2954 // This instruction modifies or uses CPSR after the one we want to
2955 // change. We can't do this transformation.
2956 return false;
2957
2958 if (I == B) {
2959 // In some cases, we scan the use-list of an instruction for an AND;
2960 // that AND is in the same BB, but may not be scheduled before the
2961 // corresponding TST. In that case, bail out.
2962 //
2963 // FIXME: We could try to reschedule the AND.
2964 return false;
2965 }
2966 } while (true);
2967
2968 // Return false if no candidates exist.
2969 if (!MI && !SubAdd)
2970 return false;
2971
2972 // If we found a SubAdd, use it as it will be closer to the CMP
2973 if (SubAdd) {
2974 MI = SubAdd;
2975 IsThumb1 = SubAddIsThumb1;
2976 }
2977
2978 // We can't use a predicated instruction - it doesn't always write the flags.
2979 if (isPredicated(MI: *MI))
2980 return false;
2981
2982 // Scan forward for the use of CPSR
2983 // When checking against MI: if it's a conditional code that requires
2984 // checking of the V bit or C bit, then this is not safe to do.
2985 // It is safe to remove CmpInstr if CPSR is redefined or killed.
2986 // If we are done with the basic block, we need to check whether CPSR is
2987 // live-out.
2988 SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
2989 OperandsToUpdate;
2990 bool isSafe = false;
2991 I = CmpInstr;
2992 E = CmpInstr.getParent()->end();
2993 while (!isSafe && ++I != E) {
2994 const MachineInstr &Instr = *I;
2995 for (unsigned IO = 0, EO = Instr.getNumOperands();
2996 !isSafe && IO != EO; ++IO) {
2997 const MachineOperand &MO = Instr.getOperand(i: IO);
2998 if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: ARM::CPSR)) {
2999 isSafe = true;
3000 break;
3001 }
3002 if (!MO.isReg() || MO.getReg() != ARM::CPSR)
3003 continue;
3004 if (MO.isDef()) {
3005 isSafe = true;
3006 break;
3007 }
3008 // Condition code is after the operand before CPSR except for VSELs.
3009 ARMCC::CondCodes CC;
3010 bool IsInstrVSel = true;
3011 switch (Instr.getOpcode()) {
3012 default:
3013 IsInstrVSel = false;
3014 CC = (ARMCC::CondCodes)Instr.getOperand(i: IO - 1).getImm();
3015 break;
3016 case ARM::VSELEQD:
3017 case ARM::VSELEQS:
3018 case ARM::VSELEQH:
3019 CC = ARMCC::EQ;
3020 break;
3021 case ARM::VSELGTD:
3022 case ARM::VSELGTS:
3023 case ARM::VSELGTH:
3024 CC = ARMCC::GT;
3025 break;
3026 case ARM::VSELGED:
3027 case ARM::VSELGES:
3028 case ARM::VSELGEH:
3029 CC = ARMCC::GE;
3030 break;
3031 case ARM::VSELVSD:
3032 case ARM::VSELVSS:
3033 case ARM::VSELVSH:
3034 CC = ARMCC::VS;
3035 break;
3036 }
3037
3038 if (SubAdd) {
3039 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3040 // on CMP needs to be updated to be based on SUB.
3041 // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3042 // needs to be modified.
3043 // Push the condition code operands to OperandsToUpdate.
3044 // If it is safe to remove CmpInstr, the condition code of these
3045 // operands will be modified.
3046 unsigned Opc = SubAdd->getOpcode();
3047 bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3048 Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3049 Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3050 Opc == ARM::tSUBi8;
3051 unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3052 if (!IsSub ||
3053 (SrcReg2 != 0 && SubAdd->getOperand(i: OpI).getReg() == SrcReg2 &&
3054 SubAdd->getOperand(i: OpI + 1).getReg() == SrcReg)) {
3055 // VSel doesn't support condition code update.
3056 if (IsInstrVSel)
3057 return false;
3058 // Ensure we can swap the condition.
3059 ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3060 if (NewCC == ARMCC::AL)
3061 return false;
3062 OperandsToUpdate.push_back(
3063 Elt: std::make_pair(x: &((*I).getOperand(i: IO - 1)), y&: NewCC));
3064 }
3065 } else {
3066 // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3067 switch (CC) {
3068 case ARMCC::EQ: // Z
3069 case ARMCC::NE: // Z
3070 case ARMCC::MI: // N
3071 case ARMCC::PL: // N
3072 case ARMCC::AL: // none
3073 // CPSR can be used multiple times, we should continue.
3074 break;
3075 case ARMCC::HS: // C
3076 case ARMCC::LO: // C
3077 case ARMCC::VS: // V
3078 case ARMCC::VC: // V
3079 case ARMCC::HI: // C Z
3080 case ARMCC::LS: // C Z
3081 case ARMCC::GE: // N V
3082 case ARMCC::LT: // N V
3083 case ARMCC::GT: // Z N V
3084 case ARMCC::LE: // Z N V
3085 // The instruction uses the V bit or C bit which is not safe.
3086 return false;
3087 }
3088 }
3089 }
3090 }
3091
3092 // If CPSR is not killed nor re-defined, we should check whether it is
3093 // live-out. If it is live-out, do not optimize.
3094 if (!isSafe) {
3095 MachineBasicBlock *MBB = CmpInstr.getParent();
3096 for (MachineBasicBlock *Succ : MBB->successors())
3097 if (Succ->isLiveIn(Reg: ARM::CPSR))
3098 return false;
3099 }
3100
3101 // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3102 // set CPSR so this is represented as an explicit output)
3103 if (!IsThumb1) {
3104 unsigned CPSRRegNum = MI->getNumExplicitOperands() - 1;
3105 MI->getOperand(i: CPSRRegNum).setReg(ARM::CPSR);
3106 MI->getOperand(i: CPSRRegNum).setIsDef(true);
3107 }
3108 assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3109 CmpInstr.eraseFromParent();
3110
3111 // Modify the condition code of operands in OperandsToUpdate.
3112 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3113 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3114 for (auto &[MO, Cond] : OperandsToUpdate)
3115 MO->setImm(Cond);
3116
3117 MI->clearRegisterDeads(Reg: ARM::CPSR);
3118
3119 return true;
3120}
3121
3122bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3123 // Do not sink MI if it might be used to optimize a redundant compare.
3124 // We heuristically only look at the instruction immediately following MI to
3125 // avoid potentially searching the entire basic block.
3126 if (isPredicated(MI))
3127 return true;
3128 MachineBasicBlock::const_iterator Next = &MI;
3129 ++Next;
3130 Register SrcReg, SrcReg2;
3131 int64_t CmpMask, CmpValue;
3132 bool IsThumb1;
3133 if (Next != MI.getParent()->end() &&
3134 analyzeCompare(MI: *Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3135 isRedundantFlagInstr(CmpI: &*Next, SrcReg, SrcReg2, ImmValue: CmpValue, OI: &MI, IsThumb1))
3136 return false;
3137 return true;
3138}
3139
3140bool ARMBaseInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3141 Register Reg,
3142 MachineRegisterInfo *MRI) const {
3143 // Fold large immediates into add, sub, or, xor.
3144 unsigned DefOpc = DefMI.getOpcode();
3145 if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm &&
3146 DefOpc != ARM::tMOVi32imm)
3147 return false;
3148 if (!DefMI.getOperand(i: 1).isImm())
3149 // Could be t2MOVi32imm @xx
3150 return false;
3151
3152 if (!MRI->hasOneNonDBGUse(RegNo: Reg))
3153 return false;
3154
3155 const MCInstrDesc &DefMCID = DefMI.getDesc();
3156 if (DefMCID.hasOptionalDef()) {
3157 unsigned NumOps = DefMCID.getNumOperands();
3158 const MachineOperand &MO = DefMI.getOperand(i: NumOps - 1);
3159 if (MO.getReg() == ARM::CPSR && !MO.isDead())
3160 // If DefMI defines CPSR and it is not dead, it's obviously not safe
3161 // to delete DefMI.
3162 return false;
3163 }
3164
3165 const MCInstrDesc &UseMCID = UseMI.getDesc();
3166 if (UseMCID.hasOptionalDef()) {
3167 unsigned NumOps = UseMCID.getNumOperands();
3168 if (UseMI.getOperand(i: NumOps - 1).getReg() == ARM::CPSR)
3169 // If the instruction sets the flag, do not attempt this optimization
3170 // since it may change the semantics of the code.
3171 return false;
3172 }
3173
3174 unsigned UseOpc = UseMI.getOpcode();
3175 unsigned NewUseOpc = 0;
3176 uint32_t ImmVal = (uint32_t)DefMI.getOperand(i: 1).getImm();
3177 uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3178 bool Commute = false;
3179 switch (UseOpc) {
3180 default: return false;
3181 case ARM::SUBrr:
3182 case ARM::ADDrr:
3183 case ARM::ORRrr:
3184 case ARM::EORrr:
3185 case ARM::t2SUBrr:
3186 case ARM::t2ADDrr:
3187 case ARM::t2ORRrr:
3188 case ARM::t2EORrr: {
3189 Commute = UseMI.getOperand(i: 2).getReg() != Reg;
3190 switch (UseOpc) {
3191 default: break;
3192 case ARM::ADDrr:
3193 case ARM::SUBrr:
3194 if (UseOpc == ARM::SUBrr && Commute)
3195 return false;
3196
3197 // ADD/SUB are special because they're essentially the same operation, so
3198 // we can handle a larger range of immediates.
3199 if (ARM_AM::isSOImmTwoPartVal(V: ImmVal))
3200 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3201 else if (ARM_AM::isSOImmTwoPartVal(V: -ImmVal)) {
3202 ImmVal = -ImmVal;
3203 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3204 } else
3205 return false;
3206 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(V: ImmVal);
3207 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(V: ImmVal);
3208 break;
3209 case ARM::ORRrr:
3210 case ARM::EORrr:
3211 if (!ARM_AM::isSOImmTwoPartVal(V: ImmVal))
3212 return false;
3213 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(V: ImmVal);
3214 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(V: ImmVal);
3215 switch (UseOpc) {
3216 default: break;
3217 case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3218 case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3219 }
3220 break;
3221 case ARM::t2ADDrr:
3222 case ARM::t2SUBrr: {
3223 if (UseOpc == ARM::t2SUBrr && Commute)
3224 return false;
3225
3226 // ADD/SUB are special because they're essentially the same operation, so
3227 // we can handle a larger range of immediates.
3228 const bool ToSP = DefMI.getOperand(i: 0).getReg() == ARM::SP;
3229 const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3230 const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3231 if (ARM_AM::isT2SOImmTwoPartVal(Imm: ImmVal))
3232 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3233 else if (ARM_AM::isT2SOImmTwoPartVal(Imm: -ImmVal)) {
3234 ImmVal = -ImmVal;
3235 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3236 } else
3237 return false;
3238 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(Imm: ImmVal);
3239 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(Imm: ImmVal);
3240 break;
3241 }
3242 case ARM::t2ORRrr:
3243 case ARM::t2EORrr:
3244 if (!ARM_AM::isT2SOImmTwoPartVal(Imm: ImmVal))
3245 return false;
3246 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(Imm: ImmVal);
3247 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(Imm: ImmVal);
3248 switch (UseOpc) {
3249 default: break;
3250 case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3251 case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3252 }
3253 break;
3254 }
3255 }
3256 }
3257
3258 unsigned OpIdx = Commute ? 2 : 1;
3259 Register Reg1 = UseMI.getOperand(i: OpIdx).getReg();
3260 bool isKill = UseMI.getOperand(i: OpIdx).isKill();
3261 const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3262 Register NewReg = MRI->createVirtualRegister(RegClass: TRC);
3263 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: NewUseOpc),
3264 DestReg: NewReg)
3265 .addReg(RegNo: Reg1, Flags: getKillRegState(B: isKill))
3266 .addImm(Val: SOImmValV1)
3267 .add(MOs: predOps(Pred: ARMCC::AL))
3268 .add(MO: condCodeOp());
3269 UseMI.setDesc(get(Opcode: NewUseOpc));
3270 UseMI.getOperand(i: 1).setReg(NewReg);
3271 UseMI.getOperand(i: 1).setIsKill();
3272 UseMI.getOperand(i: 2).ChangeToImmediate(ImmVal: SOImmValV2);
3273 DefMI.eraseFromParent();
3274 // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3275 // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3276 // Then the below code will not be needed, as the input/output register
3277 // classes will be rgpr or gprSP.
3278 // For now, we fix the UseMI operand explicitly here:
3279 switch(NewUseOpc){
3280 case ARM::t2ADDspImm:
3281 case ARM::t2SUBspImm:
3282 case ARM::t2ADDri:
3283 case ARM::t2SUBri:
3284 MRI->constrainRegClass(Reg: UseMI.getOperand(i: 0).getReg(), RC: TRC);
3285 }
3286 return true;
3287}
3288
3289static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3290 const MachineInstr &MI) {
3291 switch (MI.getOpcode()) {
3292 default: {
3293 const MCInstrDesc &Desc = MI.getDesc();
3294 int UOps = ItinData->getNumMicroOps(ItinClassIndx: Desc.getSchedClass());
3295 assert(UOps >= 0 && "bad # UOps");
3296 return UOps;
3297 }
3298
3299 case ARM::LDRrs:
3300 case ARM::LDRBrs:
3301 case ARM::STRrs:
3302 case ARM::STRBrs: {
3303 unsigned ShOpVal = MI.getOperand(i: 3).getImm();
3304 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3305 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3306 if (!isSub &&
3307 (ShImm == 0 ||
3308 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3309 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3310 return 1;
3311 return 2;
3312 }
3313
3314 case ARM::LDRH:
3315 case ARM::STRH: {
3316 if (!MI.getOperand(i: 2).getReg())
3317 return 1;
3318
3319 unsigned ShOpVal = MI.getOperand(i: 3).getImm();
3320 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3321 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3322 if (!isSub &&
3323 (ShImm == 0 ||
3324 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3325 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3326 return 1;
3327 return 2;
3328 }
3329
3330 case ARM::LDRSB:
3331 case ARM::LDRSH:
3332 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 3).getImm()) == ARM_AM::sub) ? 3 : 2;
3333
3334 case ARM::LDRSB_POST:
3335 case ARM::LDRSH_POST: {
3336 Register Rt = MI.getOperand(i: 0).getReg();
3337 Register Rm = MI.getOperand(i: 3).getReg();
3338 return (Rt == Rm) ? 4 : 3;
3339 }
3340
3341 case ARM::LDR_PRE_REG:
3342 case ARM::LDRB_PRE_REG: {
3343 Register Rt = MI.getOperand(i: 0).getReg();
3344 Register Rm = MI.getOperand(i: 3).getReg();
3345 if (Rt == Rm)
3346 return 3;
3347 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3348 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3349 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3350 if (!isSub &&
3351 (ShImm == 0 ||
3352 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3353 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3354 return 2;
3355 return 3;
3356 }
3357
3358 case ARM::STR_PRE_REG:
3359 case ARM::STRB_PRE_REG: {
3360 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3361 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3362 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3363 if (!isSub &&
3364 (ShImm == 0 ||
3365 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3366 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3367 return 2;
3368 return 3;
3369 }
3370
3371 case ARM::LDRH_PRE:
3372 case ARM::STRH_PRE: {
3373 Register Rt = MI.getOperand(i: 0).getReg();
3374 Register Rm = MI.getOperand(i: 3).getReg();
3375 if (!Rm)
3376 return 2;
3377 if (Rt == Rm)
3378 return 3;
3379 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 3 : 2;
3380 }
3381
3382 case ARM::LDR_POST_REG:
3383 case ARM::LDRB_POST_REG:
3384 case ARM::LDRH_POST: {
3385 Register Rt = MI.getOperand(i: 0).getReg();
3386 Register Rm = MI.getOperand(i: 3).getReg();
3387 return (Rt == Rm) ? 3 : 2;
3388 }
3389
3390 case ARM::LDR_PRE_IMM:
3391 case ARM::LDRB_PRE_IMM:
3392 case ARM::LDR_POST_IMM:
3393 case ARM::LDRB_POST_IMM:
3394 case ARM::STRB_POST_IMM:
3395 case ARM::STRB_POST_REG:
3396 case ARM::STRB_PRE_IMM:
3397 case ARM::STRH_POST:
3398 case ARM::STR_POST_IMM:
3399 case ARM::STR_POST_REG:
3400 case ARM::STR_PRE_IMM:
3401 return 2;
3402
3403 case ARM::LDRSB_PRE:
3404 case ARM::LDRSH_PRE: {
3405 Register Rm = MI.getOperand(i: 3).getReg();
3406 if (Rm == 0)
3407 return 3;
3408 Register Rt = MI.getOperand(i: 0).getReg();
3409 if (Rt == Rm)
3410 return 4;
3411 unsigned ShOpVal = MI.getOperand(i: 4).getImm();
3412 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
3413 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
3414 if (!isSub &&
3415 (ShImm == 0 ||
3416 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3417 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
3418 return 3;
3419 return 4;
3420 }
3421
3422 case ARM::LDRD: {
3423 Register Rt = MI.getOperand(i: 0).getReg();
3424 Register Rn = MI.getOperand(i: 2).getReg();
3425 Register Rm = MI.getOperand(i: 3).getReg();
3426 if (Rm)
3427 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 4
3428 : 3;
3429 return (Rt == Rn) ? 3 : 2;
3430 }
3431
3432 case ARM::STRD: {
3433 Register Rm = MI.getOperand(i: 3).getReg();
3434 if (Rm)
3435 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 4).getImm()) == ARM_AM::sub) ? 4
3436 : 3;
3437 return 2;
3438 }
3439
3440 case ARM::LDRD_POST:
3441 case ARM::t2LDRD_POST:
3442 return 3;
3443
3444 case ARM::STRD_POST:
3445 case ARM::t2STRD_POST:
3446 return 4;
3447
3448 case ARM::LDRD_PRE: {
3449 Register Rt = MI.getOperand(i: 0).getReg();
3450 Register Rn = MI.getOperand(i: 3).getReg();
3451 Register Rm = MI.getOperand(i: 4).getReg();
3452 if (Rm)
3453 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 5).getImm()) == ARM_AM::sub) ? 5
3454 : 4;
3455 return (Rt == Rn) ? 4 : 3;
3456 }
3457
3458 case ARM::t2LDRD_PRE: {
3459 Register Rt = MI.getOperand(i: 0).getReg();
3460 Register Rn = MI.getOperand(i: 3).getReg();
3461 return (Rt == Rn) ? 4 : 3;
3462 }
3463
3464 case ARM::STRD_PRE: {
3465 Register Rm = MI.getOperand(i: 4).getReg();
3466 if (Rm)
3467 return (ARM_AM::getAM3Op(AM3Opc: MI.getOperand(i: 5).getImm()) == ARM_AM::sub) ? 5
3468 : 4;
3469 return 3;
3470 }
3471
3472 case ARM::t2STRD_PRE:
3473 return 3;
3474
3475 case ARM::t2LDR_POST:
3476 case ARM::t2LDRB_POST:
3477 case ARM::t2LDRB_PRE:
3478 case ARM::t2LDRSBi12:
3479 case ARM::t2LDRSBi8:
3480 case ARM::t2LDRSBpci:
3481 case ARM::t2LDRSBs:
3482 case ARM::t2LDRH_POST:
3483 case ARM::t2LDRH_PRE:
3484 case ARM::t2LDRSBT:
3485 case ARM::t2LDRSB_POST:
3486 case ARM::t2LDRSB_PRE:
3487 case ARM::t2LDRSH_POST:
3488 case ARM::t2LDRSH_PRE:
3489 case ARM::t2LDRSHi12:
3490 case ARM::t2LDRSHi8:
3491 case ARM::t2LDRSHpci:
3492 case ARM::t2LDRSHs:
3493 return 2;
3494
3495 case ARM::t2LDRDi8: {
3496 Register Rt = MI.getOperand(i: 0).getReg();
3497 Register Rn = MI.getOperand(i: 2).getReg();
3498 return (Rt == Rn) ? 3 : 2;
3499 }
3500
3501 case ARM::t2STRB_POST:
3502 case ARM::t2STRB_PRE:
3503 case ARM::t2STRBs:
3504 case ARM::t2STRDi8:
3505 case ARM::t2STRH_POST:
3506 case ARM::t2STRH_PRE:
3507 case ARM::t2STRHs:
3508 case ARM::t2STR_POST:
3509 case ARM::t2STR_PRE:
3510 case ARM::t2STRs:
3511 return 2;
3512 }
3513}
3514
3515// Return the number of 32-bit words loaded by LDM or stored by STM. If this
3516// can't be easily determined return 0 (missing MachineMemOperand).
3517//
3518// FIXME: The current MachineInstr design does not support relying on machine
3519// mem operands to determine the width of a memory access. Instead, we expect
3520// the target to provide this information based on the instruction opcode and
3521// operands. However, using MachineMemOperand is the best solution now for
3522// two reasons:
3523//
3524// 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3525// operands. This is much more dangerous than using the MachineMemOperand
3526// sizes because CodeGen passes can insert/remove optional machine operands. In
3527// fact, it's totally incorrect for preRA passes and appears to be wrong for
3528// postRA passes as well.
3529//
3530// 2) getNumLDMAddresses is only used by the scheduling machine model and any
3531// machine model that calls this should handle the unknown (zero size) case.
3532//
3533// Long term, we should require a target hook that verifies MachineMemOperand
3534// sizes during MC lowering. That target hook should be local to MC lowering
3535// because we can't ensure that it is aware of other MI forms. Doing this will
3536// ensure that MachineMemOperands are correctly propagated through all passes.
3537unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3538 unsigned Size = 0;
3539 for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3540 E = MI.memoperands_end();
3541 I != E; ++I) {
3542 Size += (*I)->getSize().getValue();
3543 }
3544 // FIXME: The scheduler currently can't handle values larger than 16. But
3545 // the values can actually go up to 32 for floating-point load/store
3546 // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3547 // operations isn't right; we could end up with "extra" memory operands for
3548 // various reasons, like tail merge merging two memory operations.
3549 return std::min(a: Size / 4, b: 16U);
3550}
3551
3552static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3553 unsigned NumRegs) {
3554 unsigned UOps = 1 + NumRegs; // 1 for address computation.
3555 switch (Opc) {
3556 default:
3557 break;
3558 case ARM::VLDMDIA_UPD:
3559 case ARM::VLDMDDB_UPD:
3560 case ARM::VLDMSIA_UPD:
3561 case ARM::VLDMSDB_UPD:
3562 case ARM::VSTMDIA_UPD:
3563 case ARM::VSTMDDB_UPD:
3564 case ARM::VSTMSIA_UPD:
3565 case ARM::VSTMSDB_UPD:
3566 case ARM::LDMIA_UPD:
3567 case ARM::LDMDA_UPD:
3568 case ARM::LDMDB_UPD:
3569 case ARM::LDMIB_UPD:
3570 case ARM::STMIA_UPD:
3571 case ARM::STMDA_UPD:
3572 case ARM::STMDB_UPD:
3573 case ARM::STMIB_UPD:
3574 case ARM::tLDMIA_UPD:
3575 case ARM::tSTMIA_UPD:
3576 case ARM::t2LDMIA_UPD:
3577 case ARM::t2LDMDB_UPD:
3578 case ARM::t2STMIA_UPD:
3579 case ARM::t2STMDB_UPD:
3580 ++UOps; // One for base register writeback.
3581 break;
3582 case ARM::LDMIA_RET:
3583 case ARM::tPOP_RET:
3584 case ARM::t2LDMIA_RET:
3585 UOps += 2; // One for base reg wb, one for write to pc.
3586 break;
3587 }
3588 return UOps;
3589}
3590
3591unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3592 const MachineInstr &MI) const {
3593 if (!ItinData || ItinData->isEmpty())
3594 return 1;
3595
3596 const MCInstrDesc &Desc = MI.getDesc();
3597 unsigned Class = Desc.getSchedClass();
3598 int ItinUOps = ItinData->getNumMicroOps(ItinClassIndx: Class);
3599 if (ItinUOps >= 0) {
3600 if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3601 return getNumMicroOpsSwiftLdSt(ItinData, MI);
3602
3603 return ItinUOps;
3604 }
3605
3606 unsigned Opc = MI.getOpcode();
3607 switch (Opc) {
3608 default:
3609 llvm_unreachable("Unexpected multi-uops instruction!");
3610 case ARM::VLDMQIA:
3611 case ARM::VSTMQIA:
3612 return 2;
3613
3614 // The number of uOps for load / store multiple are determined by the number
3615 // registers.
3616 //
3617 // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3618 // same cycle. The scheduling for the first load / store must be done
3619 // separately by assuming the address is not 64-bit aligned.
3620 //
3621 // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3622 // is not 64-bit aligned, then AGU would take an extra cycle. For VFP / NEON
3623 // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3624 case ARM::VLDMDIA:
3625 case ARM::VLDMDIA_UPD:
3626 case ARM::VLDMDDB_UPD:
3627 case ARM::VLDMSIA:
3628 case ARM::VLDMSIA_UPD:
3629 case ARM::VLDMSDB_UPD:
3630 case ARM::VSTMDIA:
3631 case ARM::VSTMDIA_UPD:
3632 case ARM::VSTMDDB_UPD:
3633 case ARM::VSTMSIA:
3634 case ARM::VSTMSIA_UPD:
3635 case ARM::VSTMSDB_UPD: {
3636 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3637 return (NumRegs / 2) + (NumRegs % 2) + 1;
3638 }
3639
3640 case ARM::LDMIA_RET:
3641 case ARM::LDMIA:
3642 case ARM::LDMDA:
3643 case ARM::LDMDB:
3644 case ARM::LDMIB:
3645 case ARM::LDMIA_UPD:
3646 case ARM::LDMDA_UPD:
3647 case ARM::LDMDB_UPD:
3648 case ARM::LDMIB_UPD:
3649 case ARM::STMIA:
3650 case ARM::STMDA:
3651 case ARM::STMDB:
3652 case ARM::STMIB:
3653 case ARM::STMIA_UPD:
3654 case ARM::STMDA_UPD:
3655 case ARM::STMDB_UPD:
3656 case ARM::STMIB_UPD:
3657 case ARM::tLDMIA:
3658 case ARM::tLDMIA_UPD:
3659 case ARM::tSTMIA_UPD:
3660 case ARM::tPOP_RET:
3661 case ARM::tPOP:
3662 case ARM::tPUSH:
3663 case ARM::t2LDMIA_RET:
3664 case ARM::t2LDMIA:
3665 case ARM::t2LDMDB:
3666 case ARM::t2LDMIA_UPD:
3667 case ARM::t2LDMDB_UPD:
3668 case ARM::t2STMIA:
3669 case ARM::t2STMDB:
3670 case ARM::t2STMIA_UPD:
3671 case ARM::t2STMDB_UPD: {
3672 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3673 switch (Subtarget.getLdStMultipleTiming()) {
3674 case ARMSubtarget::SingleIssuePlusExtras:
3675 return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3676 case ARMSubtarget::SingleIssue:
3677 // Assume the worst.
3678 return NumRegs;
3679 case ARMSubtarget::DoubleIssue: {
3680 if (NumRegs < 4)
3681 return 2;
3682 // 4 registers would be issued: 2, 2.
3683 // 5 registers would be issued: 2, 2, 1.
3684 unsigned UOps = (NumRegs / 2);
3685 if (NumRegs % 2)
3686 ++UOps;
3687 return UOps;
3688 }
3689 case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3690 unsigned UOps = (NumRegs / 2);
3691 // If there are odd number of registers or if it's not 64-bit aligned,
3692 // then it takes an extra AGU (Address Generation Unit) cycle.
3693 if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3694 (*MI.memoperands_begin())->getAlign() < Align(8))
3695 ++UOps;
3696 return UOps;
3697 }
3698 }
3699 }
3700 }
3701 llvm_unreachable("Didn't find the number of microops");
3702}
3703
3704std::optional<unsigned>
3705ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3706 const MCInstrDesc &DefMCID, unsigned DefClass,
3707 unsigned DefIdx, unsigned DefAlign) const {
3708 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3709 if (RegNo <= 0)
3710 // Def is the address writeback.
3711 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3712
3713 unsigned DefCycle;
3714 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3715 // (regno / 2) + (regno % 2) + 1
3716 DefCycle = RegNo / 2 + 1;
3717 if (RegNo % 2)
3718 ++DefCycle;
3719 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3720 DefCycle = RegNo;
3721 bool isSLoad = false;
3722
3723 switch (DefMCID.getOpcode()) {
3724 default: break;
3725 case ARM::VLDMSIA:
3726 case ARM::VLDMSIA_UPD:
3727 case ARM::VLDMSDB_UPD:
3728 isSLoad = true;
3729 break;
3730 }
3731
3732 // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3733 // then it takes an extra cycle.
3734 if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3735 ++DefCycle;
3736 } else {
3737 // Assume the worst.
3738 DefCycle = RegNo + 2;
3739 }
3740
3741 return DefCycle;
3742}
3743
3744std::optional<unsigned>
3745ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3746 const MCInstrDesc &DefMCID, unsigned DefClass,
3747 unsigned DefIdx, unsigned DefAlign) const {
3748 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3749 if (RegNo <= 0)
3750 // Def is the address writeback.
3751 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3752
3753 unsigned DefCycle;
3754 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3755 // 4 registers would be issued: 1, 2, 1.
3756 // 5 registers would be issued: 1, 2, 2.
3757 DefCycle = RegNo / 2;
3758 if (DefCycle < 1)
3759 DefCycle = 1;
3760 // Result latency is issue cycle + 2: E2.
3761 DefCycle += 2;
3762 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3763 DefCycle = (RegNo / 2);
3764 // If there are odd number of registers or if it's not 64-bit aligned,
3765 // then it takes an extra AGU (Address Generation Unit) cycle.
3766 if ((RegNo % 2) || DefAlign < 8)
3767 ++DefCycle;
3768 // Result latency is AGU cycles + 2.
3769 DefCycle += 2;
3770 } else {
3771 // Assume the worst.
3772 DefCycle = RegNo + 2;
3773 }
3774
3775 return DefCycle;
3776}
3777
3778std::optional<unsigned>
3779ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3780 const MCInstrDesc &UseMCID, unsigned UseClass,
3781 unsigned UseIdx, unsigned UseAlign) const {
3782 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3783 if (RegNo <= 0)
3784 return ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3785
3786 unsigned UseCycle;
3787 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3788 // (regno / 2) + (regno % 2) + 1
3789 UseCycle = RegNo / 2 + 1;
3790 if (RegNo % 2)
3791 ++UseCycle;
3792 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3793 UseCycle = RegNo;
3794 bool isSStore = false;
3795
3796 switch (UseMCID.getOpcode()) {
3797 default: break;
3798 case ARM::VSTMSIA:
3799 case ARM::VSTMSIA_UPD:
3800 case ARM::VSTMSDB_UPD:
3801 isSStore = true;
3802 break;
3803 }
3804
3805 // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3806 // then it takes an extra cycle.
3807 if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3808 ++UseCycle;
3809 } else {
3810 // Assume the worst.
3811 UseCycle = RegNo + 2;
3812 }
3813
3814 return UseCycle;
3815}
3816
3817std::optional<unsigned>
3818ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
3819 const MCInstrDesc &UseMCID, unsigned UseClass,
3820 unsigned UseIdx, unsigned UseAlign) const {
3821 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3822 if (RegNo <= 0)
3823 return ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3824
3825 unsigned UseCycle;
3826 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3827 UseCycle = RegNo / 2;
3828 if (UseCycle < 2)
3829 UseCycle = 2;
3830 // Read in E3.
3831 UseCycle += 2;
3832 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3833 UseCycle = (RegNo / 2);
3834 // If there are odd number of registers or if it's not 64-bit aligned,
3835 // then it takes an extra AGU (Address Generation Unit) cycle.
3836 if ((RegNo % 2) || UseAlign < 8)
3837 ++UseCycle;
3838 } else {
3839 // Assume the worst.
3840 UseCycle = 1;
3841 }
3842 return UseCycle;
3843}
3844
3845std::optional<unsigned> ARMBaseInstrInfo::getOperandLatency(
3846 const InstrItineraryData *ItinData, const MCInstrDesc &DefMCID,
3847 unsigned DefIdx, unsigned DefAlign, const MCInstrDesc &UseMCID,
3848 unsigned UseIdx, unsigned UseAlign) const {
3849 unsigned DefClass = DefMCID.getSchedClass();
3850 unsigned UseClass = UseMCID.getSchedClass();
3851
3852 if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
3853 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
3854
3855 // This may be a def / use of a variable_ops instruction, the operand
3856 // latency might be determinable dynamically. Let the target try to
3857 // figure it out.
3858 std::optional<unsigned> DefCycle;
3859 bool LdmBypass = false;
3860 switch (DefMCID.getOpcode()) {
3861 default:
3862 DefCycle = ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
3863 break;
3864
3865 case ARM::VLDMDIA:
3866 case ARM::VLDMDIA_UPD:
3867 case ARM::VLDMDDB_UPD:
3868 case ARM::VLDMSIA:
3869 case ARM::VLDMSIA_UPD:
3870 case ARM::VLDMSDB_UPD:
3871 DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3872 break;
3873
3874 case ARM::LDMIA_RET:
3875 case ARM::LDMIA:
3876 case ARM::LDMDA:
3877 case ARM::LDMDB:
3878 case ARM::LDMIB:
3879 case ARM::LDMIA_UPD:
3880 case ARM::LDMDA_UPD:
3881 case ARM::LDMDB_UPD:
3882 case ARM::LDMIB_UPD:
3883 case ARM::tLDMIA:
3884 case ARM::tLDMIA_UPD:
3885 case ARM::tPUSH:
3886 case ARM::t2LDMIA_RET:
3887 case ARM::t2LDMIA:
3888 case ARM::t2LDMDB:
3889 case ARM::t2LDMIA_UPD:
3890 case ARM::t2LDMDB_UPD:
3891 LdmBypass = true;
3892 DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
3893 break;
3894 }
3895
3896 if (!DefCycle)
3897 // We can't seem to determine the result latency of the def, assume it's 2.
3898 DefCycle = 2;
3899
3900 std::optional<unsigned> UseCycle;
3901 switch (UseMCID.getOpcode()) {
3902 default:
3903 UseCycle = ItinData->getOperandCycle(ItinClassIndx: UseClass, OperandIdx: UseIdx);
3904 break;
3905
3906 case ARM::VSTMDIA:
3907 case ARM::VSTMDIA_UPD:
3908 case ARM::VSTMDDB_UPD:
3909 case ARM::VSTMSIA:
3910 case ARM::VSTMSIA_UPD:
3911 case ARM::VSTMSDB_UPD:
3912 UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3913 break;
3914
3915 case ARM::STMIA:
3916 case ARM::STMDA:
3917 case ARM::STMDB:
3918 case ARM::STMIB:
3919 case ARM::STMIA_UPD:
3920 case ARM::STMDA_UPD:
3921 case ARM::STMDB_UPD:
3922 case ARM::STMIB_UPD:
3923 case ARM::tSTMIA_UPD:
3924 case ARM::tPOP_RET:
3925 case ARM::tPOP:
3926 case ARM::t2STMIA:
3927 case ARM::t2STMDB:
3928 case ARM::t2STMIA_UPD:
3929 case ARM::t2STMDB_UPD:
3930 UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
3931 break;
3932 }
3933
3934 if (!UseCycle)
3935 // Assume it's read in the first stage.
3936 UseCycle = 1;
3937
3938 if (UseCycle > *DefCycle + 1)
3939 return std::nullopt;
3940
3941 UseCycle = *DefCycle - *UseCycle + 1;
3942 if (UseCycle > 0u) {
3943 if (LdmBypass) {
3944 // It's a variable_ops instruction so we can't use DefIdx here. Just use
3945 // first def operand.
3946 if (ItinData->hasPipelineForwarding(DefClass, DefIdx: DefMCID.getNumOperands()-1,
3947 UseClass, UseIdx))
3948 UseCycle = *UseCycle - 1;
3949 } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
3950 UseClass, UseIdx)) {
3951 UseCycle = *UseCycle - 1;
3952 }
3953 }
3954
3955 return UseCycle;
3956}
3957
3958static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
3959 const MachineInstr *MI, unsigned Reg,
3960 unsigned &DefIdx, unsigned &Dist) {
3961 Dist = 0;
3962
3963 MachineBasicBlock::const_iterator I = MI; ++I;
3964 MachineBasicBlock::const_instr_iterator II = std::prev(x: I.getInstrIterator());
3965 assert(II->isInsideBundle() && "Empty bundle?");
3966
3967 int Idx = -1;
3968 while (II->isInsideBundle()) {
3969 Idx = II->findRegisterDefOperandIdx(Reg, TRI, isDead: false, Overlap: true);
3970 if (Idx != -1)
3971 break;
3972 --II;
3973 ++Dist;
3974 }
3975
3976 assert(Idx != -1 && "Cannot find bundled definition!");
3977 DefIdx = Idx;
3978 return &*II;
3979}
3980
3981static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
3982 const MachineInstr &MI, unsigned Reg,
3983 unsigned &UseIdx, unsigned &Dist) {
3984 Dist = 0;
3985
3986 MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
3987 assert(II->isInsideBundle() && "Empty bundle?");
3988 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
3989
3990 // FIXME: This doesn't properly handle multiple uses.
3991 int Idx = -1;
3992 while (II != E && II->isInsideBundle()) {
3993 Idx = II->findRegisterUseOperandIdx(Reg, TRI, isKill: false);
3994 if (Idx != -1)
3995 break;
3996 if (II->getOpcode() != ARM::t2IT)
3997 ++Dist;
3998 ++II;
3999 }
4000
4001 if (Idx == -1) {
4002 Dist = 0;
4003 return nullptr;
4004 }
4005
4006 UseIdx = Idx;
4007 return &*II;
4008}
4009
4010/// Return the number of cycles to add to (or subtract from) the static
4011/// itinerary based on the def opcode and alignment. The caller will ensure that
4012/// adjusted latency is at least one cycle.
4013static int adjustDefLatency(const ARMSubtarget &Subtarget,
4014 const MachineInstr &DefMI,
4015 const MCInstrDesc &DefMCID, unsigned DefAlign) {
4016 int Adjust = 0;
4017 if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4018 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4019 // variants are one cycle cheaper.
4020 switch (DefMCID.getOpcode()) {
4021 default: break;
4022 case ARM::LDRrs:
4023 case ARM::LDRBrs: {
4024 unsigned ShOpVal = DefMI.getOperand(i: 3).getImm();
4025 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4026 if (ShImm == 0 ||
4027 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4028 --Adjust;
4029 break;
4030 }
4031 case ARM::t2LDRs:
4032 case ARM::t2LDRBs:
4033 case ARM::t2LDRHs:
4034 case ARM::t2LDRSHs: {
4035 // Thumb2 mode: lsl only.
4036 unsigned ShAmt = DefMI.getOperand(i: 3).getImm();
4037 if (ShAmt == 0 || ShAmt == 2)
4038 --Adjust;
4039 break;
4040 }
4041 }
4042 } else if (Subtarget.isSwift()) {
4043 // FIXME: Properly handle all of the latency adjustments for address
4044 // writeback.
4045 switch (DefMCID.getOpcode()) {
4046 default: break;
4047 case ARM::LDRrs:
4048 case ARM::LDRBrs: {
4049 unsigned ShOpVal = DefMI.getOperand(i: 3).getImm();
4050 bool isSub = ARM_AM::getAM2Op(AM2Opc: ShOpVal) == ARM_AM::sub;
4051 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4052 if (!isSub &&
4053 (ShImm == 0 ||
4054 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4055 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl)))
4056 Adjust -= 2;
4057 else if (!isSub &&
4058 ShImm == 1 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsr)
4059 --Adjust;
4060 break;
4061 }
4062 case ARM::t2LDRs:
4063 case ARM::t2LDRBs:
4064 case ARM::t2LDRHs:
4065 case ARM::t2LDRSHs: {
4066 // Thumb2 mode: lsl only.
4067 unsigned ShAmt = DefMI.getOperand(i: 3).getImm();
4068 if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4069 Adjust -= 2;
4070 break;
4071 }
4072 }
4073 }
4074
4075 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4076 switch (DefMCID.getOpcode()) {
4077 default: break;
4078 case ARM::VLD1q8:
4079 case ARM::VLD1q16:
4080 case ARM::VLD1q32:
4081 case ARM::VLD1q64:
4082 case ARM::VLD1q8wb_fixed:
4083 case ARM::VLD1q16wb_fixed:
4084 case ARM::VLD1q32wb_fixed:
4085 case ARM::VLD1q64wb_fixed:
4086 case ARM::VLD1q8wb_register:
4087 case ARM::VLD1q16wb_register:
4088 case ARM::VLD1q32wb_register:
4089 case ARM::VLD1q64wb_register:
4090 case ARM::VLD2d8:
4091 case ARM::VLD2d16:
4092 case ARM::VLD2d32:
4093 case ARM::VLD2q8:
4094 case ARM::VLD2q16:
4095 case ARM::VLD2q32:
4096 case ARM::VLD2d8wb_fixed:
4097 case ARM::VLD2d16wb_fixed:
4098 case ARM::VLD2d32wb_fixed:
4099 case ARM::VLD2q8wb_fixed:
4100 case ARM::VLD2q16wb_fixed:
4101 case ARM::VLD2q32wb_fixed:
4102 case ARM::VLD2d8wb_register:
4103 case ARM::VLD2d16wb_register:
4104 case ARM::VLD2d32wb_register:
4105 case ARM::VLD2q8wb_register:
4106 case ARM::VLD2q16wb_register:
4107 case ARM::VLD2q32wb_register:
4108 case ARM::VLD3d8:
4109 case ARM::VLD3d16:
4110 case ARM::VLD3d32:
4111 case ARM::VLD1d64T:
4112 case ARM::VLD3d8_UPD:
4113 case ARM::VLD3d16_UPD:
4114 case ARM::VLD3d32_UPD:
4115 case ARM::VLD1d64Twb_fixed:
4116 case ARM::VLD1d64Twb_register:
4117 case ARM::VLD3q8_UPD:
4118 case ARM::VLD3q16_UPD:
4119 case ARM::VLD3q32_UPD:
4120 case ARM::VLD4d8:
4121 case ARM::VLD4d16:
4122 case ARM::VLD4d32:
4123 case ARM::VLD1d64Q:
4124 case ARM::VLD4d8_UPD:
4125 case ARM::VLD4d16_UPD:
4126 case ARM::VLD4d32_UPD:
4127 case ARM::VLD1d64Qwb_fixed:
4128 case ARM::VLD1d64Qwb_register:
4129 case ARM::VLD4q8_UPD:
4130 case ARM::VLD4q16_UPD:
4131 case ARM::VLD4q32_UPD:
4132 case ARM::VLD1DUPq8:
4133 case ARM::VLD1DUPq16:
4134 case ARM::VLD1DUPq32:
4135 case ARM::VLD1DUPq8wb_fixed:
4136 case ARM::VLD1DUPq16wb_fixed:
4137 case ARM::VLD1DUPq32wb_fixed:
4138 case ARM::VLD1DUPq8wb_register:
4139 case ARM::VLD1DUPq16wb_register:
4140 case ARM::VLD1DUPq32wb_register:
4141 case ARM::VLD2DUPd8:
4142 case ARM::VLD2DUPd16:
4143 case ARM::VLD2DUPd32:
4144 case ARM::VLD2DUPd8wb_fixed:
4145 case ARM::VLD2DUPd16wb_fixed:
4146 case ARM::VLD2DUPd32wb_fixed:
4147 case ARM::VLD2DUPd8wb_register:
4148 case ARM::VLD2DUPd16wb_register:
4149 case ARM::VLD2DUPd32wb_register:
4150 case ARM::VLD4DUPd8:
4151 case ARM::VLD4DUPd16:
4152 case ARM::VLD4DUPd32:
4153 case ARM::VLD4DUPd8_UPD:
4154 case ARM::VLD4DUPd16_UPD:
4155 case ARM::VLD4DUPd32_UPD:
4156 case ARM::VLD1LNd8:
4157 case ARM::VLD1LNd16:
4158 case ARM::VLD1LNd32:
4159 case ARM::VLD1LNd8_UPD:
4160 case ARM::VLD1LNd16_UPD:
4161 case ARM::VLD1LNd32_UPD:
4162 case ARM::VLD2LNd8:
4163 case ARM::VLD2LNd16:
4164 case ARM::VLD2LNd32:
4165 case ARM::VLD2LNq16:
4166 case ARM::VLD2LNq32:
4167 case ARM::VLD2LNd8_UPD:
4168 case ARM::VLD2LNd16_UPD:
4169 case ARM::VLD2LNd32_UPD:
4170 case ARM::VLD2LNq16_UPD:
4171 case ARM::VLD2LNq32_UPD:
4172 case ARM::VLD4LNd8:
4173 case ARM::VLD4LNd16:
4174 case ARM::VLD4LNd32:
4175 case ARM::VLD4LNq16:
4176 case ARM::VLD4LNq32:
4177 case ARM::VLD4LNd8_UPD:
4178 case ARM::VLD4LNd16_UPD:
4179 case ARM::VLD4LNd32_UPD:
4180 case ARM::VLD4LNq16_UPD:
4181 case ARM::VLD4LNq32_UPD:
4182 // If the address is not 64-bit aligned, the latencies of these
4183 // instructions increases by one.
4184 ++Adjust;
4185 break;
4186 }
4187 }
4188 return Adjust;
4189}
4190
4191std::optional<unsigned> ARMBaseInstrInfo::getOperandLatency(
4192 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4193 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
4194 // No operand latency. The caller may fall back to getInstrLatency.
4195 if (!ItinData || ItinData->isEmpty())
4196 return std::nullopt;
4197
4198 const MachineOperand &DefMO = DefMI.getOperand(i: DefIdx);
4199 Register Reg = DefMO.getReg();
4200
4201 const MachineInstr *ResolvedDefMI = &DefMI;
4202 unsigned DefAdj = 0;
4203 if (DefMI.isBundle())
4204 ResolvedDefMI =
4205 getBundledDefMI(TRI: &getRegisterInfo(), MI: &DefMI, Reg, DefIdx, Dist&: DefAdj);
4206 if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4207 ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4208 return 1;
4209 }
4210
4211 const MachineInstr *ResolvedUseMI = &UseMI;
4212 unsigned UseAdj = 0;
4213 if (UseMI.isBundle()) {
4214 ResolvedUseMI =
4215 getBundledUseMI(TRI: &getRegisterInfo(), MI: UseMI, Reg, UseIdx, Dist&: UseAdj);
4216 if (!ResolvedUseMI)
4217 return std::nullopt;
4218 }
4219
4220 return getOperandLatencyImpl(
4221 ItinData, DefMI: *ResolvedDefMI, DefIdx, DefMCID: ResolvedDefMI->getDesc(), DefAdj, DefMO,
4222 Reg, UseMI: *ResolvedUseMI, UseIdx, UseMCID: ResolvedUseMI->getDesc(), UseAdj);
4223}
4224
4225std::optional<unsigned> ARMBaseInstrInfo::getOperandLatencyImpl(
4226 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4227 unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4228 const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4229 unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4230 if (Reg == ARM::CPSR) {
4231 if (DefMI.getOpcode() == ARM::FMSTAT) {
4232 // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4233 return Subtarget.isLikeA9() ? 1 : 20;
4234 }
4235
4236 // CPSR set and branch can be paired in the same cycle.
4237 if (UseMI.isBranch())
4238 return 0;
4239
4240 // Otherwise it takes the instruction latency (generally one).
4241 unsigned Latency = getInstrLatency(ItinData, MI: DefMI);
4242
4243 // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4244 // its uses. Instructions which are otherwise scheduled between them may
4245 // incur a code size penalty (not able to use the CPSR setting 16-bit
4246 // instructions).
4247 if (Latency > 0 && Subtarget.isThumb2()) {
4248 const MachineFunction *MF = DefMI.getParent()->getParent();
4249 if (MF->getFunction().hasOptSize())
4250 --Latency;
4251 }
4252 return Latency;
4253 }
4254
4255 if (DefMO.isImplicit() || UseMI.getOperand(i: UseIdx).isImplicit())
4256 return std::nullopt;
4257
4258 unsigned DefAlign = DefMI.hasOneMemOperand()
4259 ? (*DefMI.memoperands_begin())->getAlign().value()
4260 : 0;
4261 unsigned UseAlign = UseMI.hasOneMemOperand()
4262 ? (*UseMI.memoperands_begin())->getAlign().value()
4263 : 0;
4264
4265 // Get the itinerary's latency if possible, and handle variable_ops.
4266 std::optional<unsigned> Latency = getOperandLatency(
4267 ItinData, DefMCID, DefIdx, DefAlign, UseMCID, UseIdx, UseAlign);
4268 // Unable to find operand latency. The caller may resort to getInstrLatency.
4269 if (!Latency)
4270 return std::nullopt;
4271
4272 // Adjust for IT block position.
4273 int Adj = DefAdj + UseAdj;
4274
4275 // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4276 Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4277 if (Adj >= 0 || (int)*Latency > -Adj) {
4278 return *Latency + Adj;
4279 }
4280 // Return the itinerary latency, which may be zero but not less than zero.
4281 return Latency;
4282}
4283
4284std::optional<unsigned>
4285ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4286 SDNode *DefNode, unsigned DefIdx,
4287 SDNode *UseNode, unsigned UseIdx) const {
4288 if (!DefNode->isMachineOpcode())
4289 return 1;
4290
4291 const MCInstrDesc &DefMCID = get(Opcode: DefNode->getMachineOpcode());
4292
4293 if (isZeroCost(Opcode: DefMCID.Opcode))
4294 return 0;
4295
4296 if (!ItinData || ItinData->isEmpty())
4297 return DefMCID.mayLoad() ? 3 : 1;
4298
4299 if (!UseNode->isMachineOpcode()) {
4300 std::optional<unsigned> Latency =
4301 ItinData->getOperandCycle(ItinClassIndx: DefMCID.getSchedClass(), OperandIdx: DefIdx);
4302 int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4303 int Threshold = 1 + Adj;
4304 return !Latency || Latency <= (unsigned)Threshold ? 1 : *Latency - Adj;
4305 }
4306
4307 const MCInstrDesc &UseMCID = get(Opcode: UseNode->getMachineOpcode());
4308 auto *DefMN = cast<MachineSDNode>(Val: DefNode);
4309 unsigned DefAlign = !DefMN->memoperands_empty()
4310 ? (*DefMN->memoperands_begin())->getAlign().value()
4311 : 0;
4312 auto *UseMN = cast<MachineSDNode>(Val: UseNode);
4313 unsigned UseAlign = !UseMN->memoperands_empty()
4314 ? (*UseMN->memoperands_begin())->getAlign().value()
4315 : 0;
4316 std::optional<unsigned> Latency = getOperandLatency(
4317 ItinData, DefMCID, DefIdx, DefAlign, UseMCID, UseIdx, UseAlign);
4318 if (!Latency)
4319 return std::nullopt;
4320
4321 if (Latency > 1U &&
4322 (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4323 Subtarget.isCortexA7())) {
4324 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4325 // variants are one cycle cheaper.
4326 switch (DefMCID.getOpcode()) {
4327 default: break;
4328 case ARM::LDRrs:
4329 case ARM::LDRBrs: {
4330 unsigned ShOpVal = DefNode->getConstantOperandVal(Num: 2);
4331 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4332 if (ShImm == 0 ||
4333 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4334 Latency = *Latency - 1;
4335 break;
4336 }
4337 case ARM::t2LDRs:
4338 case ARM::t2LDRBs:
4339 case ARM::t2LDRHs:
4340 case ARM::t2LDRSHs: {
4341 // Thumb2 mode: lsl only.
4342 unsigned ShAmt = DefNode->getConstantOperandVal(Num: 2);
4343 if (ShAmt == 0 || ShAmt == 2)
4344 Latency = *Latency - 1;
4345 break;
4346 }
4347 }
4348 } else if (DefIdx == 0 && Latency > 2U && Subtarget.isSwift()) {
4349 // FIXME: Properly handle all of the latency adjustments for address
4350 // writeback.
4351 switch (DefMCID.getOpcode()) {
4352 default: break;
4353 case ARM::LDRrs:
4354 case ARM::LDRBrs: {
4355 unsigned ShOpVal = DefNode->getConstantOperandVal(Num: 2);
4356 unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc: ShOpVal);
4357 if (ShImm == 0 ||
4358 ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4359 ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsl))
4360 Latency = *Latency - 2;
4361 else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(AM2Opc: ShOpVal) == ARM_AM::lsr)
4362 Latency = *Latency - 1;
4363 break;
4364 }
4365 case ARM::t2LDRs:
4366 case ARM::t2LDRBs:
4367 case ARM::t2LDRHs:
4368 case ARM::t2LDRSHs:
4369 // Thumb2 mode: lsl 0-3 only.
4370 Latency = *Latency - 2;
4371 break;
4372 }
4373 }
4374
4375 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4376 switch (DefMCID.getOpcode()) {
4377 default: break;
4378 case ARM::VLD1q8:
4379 case ARM::VLD1q16:
4380 case ARM::VLD1q32:
4381 case ARM::VLD1q64:
4382 case ARM::VLD1q8wb_register:
4383 case ARM::VLD1q16wb_register:
4384 case ARM::VLD1q32wb_register:
4385 case ARM::VLD1q64wb_register:
4386 case ARM::VLD1q8wb_fixed:
4387 case ARM::VLD1q16wb_fixed:
4388 case ARM::VLD1q32wb_fixed:
4389 case ARM::VLD1q64wb_fixed:
4390 case ARM::VLD2d8:
4391 case ARM::VLD2d16:
4392 case ARM::VLD2d32:
4393 case ARM::VLD2q8Pseudo:
4394 case ARM::VLD2q16Pseudo:
4395 case ARM::VLD2q32Pseudo:
4396 case ARM::VLD2d8wb_fixed:
4397 case ARM::VLD2d16wb_fixed:
4398 case ARM::VLD2d32wb_fixed:
4399 case ARM::VLD2q8PseudoWB_fixed:
4400 case ARM::VLD2q16PseudoWB_fixed:
4401 case ARM::VLD2q32PseudoWB_fixed:
4402 case ARM::VLD2d8wb_register:
4403 case ARM::VLD2d16wb_register:
4404 case ARM::VLD2d32wb_register:
4405 case ARM::VLD2q8PseudoWB_register:
4406 case ARM::VLD2q16PseudoWB_register:
4407 case ARM::VLD2q32PseudoWB_register:
4408 case ARM::VLD3d8Pseudo:
4409 case ARM::VLD3d16Pseudo:
4410 case ARM::VLD3d32Pseudo:
4411 case ARM::VLD1d8TPseudo:
4412 case ARM::VLD1d16TPseudo:
4413 case ARM::VLD1d32TPseudo:
4414 case ARM::VLD1d64TPseudo:
4415 case ARM::VLD1d64TPseudoWB_fixed:
4416 case ARM::VLD1d64TPseudoWB_register:
4417 case ARM::VLD3d8Pseudo_UPD:
4418 case ARM::VLD3d16Pseudo_UPD:
4419 case ARM::VLD3d32Pseudo_UPD:
4420 case ARM::VLD3q8Pseudo_UPD:
4421 case ARM::VLD3q16Pseudo_UPD:
4422 case ARM::VLD3q32Pseudo_UPD:
4423 case ARM::VLD3q8oddPseudo:
4424 case ARM::VLD3q16oddPseudo:
4425 case ARM::VLD3q32oddPseudo:
4426 case ARM::VLD3q8oddPseudo_UPD:
4427 case ARM::VLD3q16oddPseudo_UPD:
4428 case ARM::VLD3q32oddPseudo_UPD:
4429 case ARM::VLD4d8Pseudo:
4430 case ARM::VLD4d16Pseudo:
4431 case ARM::VLD4d32Pseudo:
4432 case ARM::VLD1d8QPseudo:
4433 case ARM::VLD1d16QPseudo:
4434 case ARM::VLD1d32QPseudo:
4435 case ARM::VLD1d64QPseudo:
4436 case ARM::VLD1d64QPseudoWB_fixed:
4437 case ARM::VLD1d64QPseudoWB_register:
4438 case ARM::VLD1q8HighQPseudo:
4439 case ARM::VLD1q8LowQPseudo_UPD:
4440 case ARM::VLD1q8HighTPseudo:
4441 case ARM::VLD1q8LowTPseudo_UPD:
4442 case ARM::VLD1q16HighQPseudo:
4443 case ARM::VLD1q16LowQPseudo_UPD:
4444 case ARM::VLD1q16HighTPseudo:
4445 case ARM::VLD1q16LowTPseudo_UPD:
4446 case ARM::VLD1q32HighQPseudo:
4447 case ARM::VLD1q32LowQPseudo_UPD:
4448 case ARM::VLD1q32HighTPseudo:
4449 case ARM::VLD1q32LowTPseudo_UPD:
4450 case ARM::VLD1q64HighQPseudo:
4451 case ARM::VLD1q64LowQPseudo_UPD:
4452 case ARM::VLD1q64HighTPseudo:
4453 case ARM::VLD1q64LowTPseudo_UPD:
4454 case ARM::VLD4d8Pseudo_UPD:
4455 case ARM::VLD4d16Pseudo_UPD:
4456 case ARM::VLD4d32Pseudo_UPD:
4457 case ARM::VLD4q8Pseudo_UPD:
4458 case ARM::VLD4q16Pseudo_UPD:
4459 case ARM::VLD4q32Pseudo_UPD:
4460 case ARM::VLD4q8oddPseudo:
4461 case ARM::VLD4q16oddPseudo:
4462 case ARM::VLD4q32oddPseudo:
4463 case ARM::VLD4q8oddPseudo_UPD:
4464 case ARM::VLD4q16oddPseudo_UPD:
4465 case ARM::VLD4q32oddPseudo_UPD:
4466 case ARM::VLD1DUPq8:
4467 case ARM::VLD1DUPq16:
4468 case ARM::VLD1DUPq32:
4469 case ARM::VLD1DUPq8wb_fixed:
4470 case ARM::VLD1DUPq16wb_fixed:
4471 case ARM::VLD1DUPq32wb_fixed:
4472 case ARM::VLD1DUPq8wb_register:
4473 case ARM::VLD1DUPq16wb_register:
4474 case ARM::VLD1DUPq32wb_register:
4475 case ARM::VLD2DUPd8:
4476 case ARM::VLD2DUPd16:
4477 case ARM::VLD2DUPd32:
4478 case ARM::VLD2DUPd8wb_fixed:
4479 case ARM::VLD2DUPd16wb_fixed:
4480 case ARM::VLD2DUPd32wb_fixed:
4481 case ARM::VLD2DUPd8wb_register:
4482 case ARM::VLD2DUPd16wb_register:
4483 case ARM::VLD2DUPd32wb_register:
4484 case ARM::VLD2DUPq8EvenPseudo:
4485 case ARM::VLD2DUPq8OddPseudo:
4486 case ARM::VLD2DUPq16EvenPseudo:
4487 case ARM::VLD2DUPq16OddPseudo:
4488 case ARM::VLD2DUPq32EvenPseudo:
4489 case ARM::VLD2DUPq32OddPseudo:
4490 case ARM::VLD3DUPq8EvenPseudo:
4491 case ARM::VLD3DUPq8OddPseudo:
4492 case ARM::VLD3DUPq16EvenPseudo:
4493 case ARM::VLD3DUPq16OddPseudo:
4494 case ARM::VLD3DUPq32EvenPseudo:
4495 case ARM::VLD3DUPq32OddPseudo:
4496 case ARM::VLD4DUPd8Pseudo:
4497 case ARM::VLD4DUPd16Pseudo:
4498 case ARM::VLD4DUPd32Pseudo:
4499 case ARM::VLD4DUPd8Pseudo_UPD:
4500 case ARM::VLD4DUPd16Pseudo_UPD:
4501 case ARM::VLD4DUPd32Pseudo_UPD:
4502 case ARM::VLD4DUPq8EvenPseudo:
4503 case ARM::VLD4DUPq8OddPseudo:
4504 case ARM::VLD4DUPq16EvenPseudo:
4505 case ARM::VLD4DUPq16OddPseudo:
4506 case ARM::VLD4DUPq32EvenPseudo:
4507 case ARM::VLD4DUPq32OddPseudo:
4508 case ARM::VLD1LNq8Pseudo:
4509 case ARM::VLD1LNq16Pseudo:
4510 case ARM::VLD1LNq32Pseudo:
4511 case ARM::VLD1LNq8Pseudo_UPD:
4512 case ARM::VLD1LNq16Pseudo_UPD:
4513 case ARM::VLD1LNq32Pseudo_UPD:
4514 case ARM::VLD2LNd8Pseudo:
4515 case ARM::VLD2LNd16Pseudo:
4516 case ARM::VLD2LNd32Pseudo:
4517 case ARM::VLD2LNq16Pseudo:
4518 case ARM::VLD2LNq32Pseudo:
4519 case ARM::VLD2LNd8Pseudo_UPD:
4520 case ARM::VLD2LNd16Pseudo_UPD:
4521 case ARM::VLD2LNd32Pseudo_UPD:
4522 case ARM::VLD2LNq16Pseudo_UPD:
4523 case ARM::VLD2LNq32Pseudo_UPD:
4524 case ARM::VLD4LNd8Pseudo:
4525 case ARM::VLD4LNd16Pseudo:
4526 case ARM::VLD4LNd32Pseudo:
4527 case ARM::VLD4LNq16Pseudo:
4528 case ARM::VLD4LNq32Pseudo:
4529 case ARM::VLD4LNd8Pseudo_UPD:
4530 case ARM::VLD4LNd16Pseudo_UPD:
4531 case ARM::VLD4LNd32Pseudo_UPD:
4532 case ARM::VLD4LNq16Pseudo_UPD:
4533 case ARM::VLD4LNq32Pseudo_UPD:
4534 // If the address is not 64-bit aligned, the latencies of these
4535 // instructions increases by one.
4536 Latency = *Latency + 1;
4537 break;
4538 }
4539
4540 return Latency;
4541}
4542
4543unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4544 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4545 MI.isImplicitDef())
4546 return 0;
4547
4548 if (MI.isBundle())
4549 return 0;
4550
4551 const MCInstrDesc &MCID = MI.getDesc();
4552
4553 if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(Reg: ARM::CPSR) &&
4554 !Subtarget.cheapPredicableCPSRDef())) {
4555 // When predicated, CPSR is an additional source operand for CPSR updating
4556 // instructions, this apparently increases their latencies.
4557 return 1;
4558 }
4559 return 0;
4560}
4561
4562unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4563 const MachineInstr &MI,
4564 unsigned *PredCost) const {
4565 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4566 MI.isImplicitDef())
4567 return 1;
4568
4569 // An instruction scheduler typically runs on unbundled instructions, however
4570 // other passes may query the latency of a bundled instruction.
4571 if (MI.isBundle()) {
4572 unsigned Latency = 0;
4573 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4574 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4575 while (++I != E && I->isInsideBundle()) {
4576 if (I->getOpcode() != ARM::t2IT)
4577 Latency += getInstrLatency(ItinData, MI: *I, PredCost);
4578 }
4579 return Latency;
4580 }
4581
4582 const MCInstrDesc &MCID = MI.getDesc();
4583 if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(Reg: ARM::CPSR) &&
4584 !Subtarget.cheapPredicableCPSRDef()))) {
4585 // When predicated, CPSR is an additional source operand for CPSR updating
4586 // instructions, this apparently increases their latencies.
4587 *PredCost = 1;
4588 }
4589 // Be sure to call getStageLatency for an empty itinerary in case it has a
4590 // valid MinLatency property.
4591 if (!ItinData)
4592 return MI.mayLoad() ? 3 : 1;
4593
4594 unsigned Class = MCID.getSchedClass();
4595
4596 // For instructions with variable uops, use uops as latency.
4597 if (!ItinData->isEmpty() && ItinData->getNumMicroOps(ItinClassIndx: Class) < 0)
4598 return getNumMicroOps(ItinData, MI);
4599
4600 // For the common case, fall back on the itinerary's latency.
4601 unsigned Latency = ItinData->getStageLatency(ItinClassIndx: Class);
4602
4603 // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4604 unsigned DefAlign =
4605 MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4606 int Adj = adjustDefLatency(Subtarget, DefMI: MI, DefMCID: MCID, DefAlign);
4607 if (Adj >= 0 || (int)Latency > -Adj) {
4608 return Latency + Adj;
4609 }
4610 return Latency;
4611}
4612
4613unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4614 SDNode *Node) const {
4615 if (!Node->isMachineOpcode())
4616 return 1;
4617
4618 if (!ItinData || ItinData->isEmpty())
4619 return 1;
4620
4621 unsigned Opcode = Node->getMachineOpcode();
4622 switch (Opcode) {
4623 default:
4624 return ItinData->getStageLatency(ItinClassIndx: get(Opcode).getSchedClass());
4625 case ARM::VLDMQIA:
4626 case ARM::VSTMQIA:
4627 return 2;
4628 }
4629}
4630
4631bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4632 const MachineRegisterInfo *MRI,
4633 const MachineInstr &DefMI,
4634 unsigned DefIdx,
4635 const MachineInstr &UseMI,
4636 unsigned UseIdx) const {
4637 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4638 unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4639 if (Subtarget.nonpipelinedVFP() &&
4640 (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4641 return true;
4642
4643 // Hoist VFP / NEON instructions with 4 or higher latency.
4644 unsigned Latency =
4645 SchedModel.computeOperandLatency(DefMI: &DefMI, DefOperIdx: DefIdx, UseMI: &UseMI, UseOperIdx: UseIdx);
4646 if (Latency <= 3)
4647 return false;
4648 return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4649 UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4650}
4651
4652bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4653 const MachineInstr &DefMI,
4654 unsigned DefIdx) const {
4655 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4656 if (!ItinData || ItinData->isEmpty())
4657 return false;
4658
4659 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4660 if (DDomain == ARMII::DomainGeneral) {
4661 unsigned DefClass = DefMI.getDesc().getSchedClass();
4662 std::optional<unsigned> DefCycle =
4663 ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
4664 return DefCycle && DefCycle <= 2U;
4665 }
4666 return false;
4667}
4668
4669bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4670 StringRef &ErrInfo) const {
4671 if (convertAddSubFlagsOpcode(OldOpc: MI.getOpcode())) {
4672 ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4673 return false;
4674 }
4675 if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4676 // Make sure we don't generate a lo-lo mov that isn't supported.
4677 if (!ARM::hGPRRegClass.contains(Reg: MI.getOperand(i: 0).getReg()) &&
4678 !ARM::hGPRRegClass.contains(Reg: MI.getOperand(i: 1).getReg())) {
4679 ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4680 return false;
4681 }
4682 }
4683 if (MI.getOpcode() == ARM::tPUSH ||
4684 MI.getOpcode() == ARM::tPOP ||
4685 MI.getOpcode() == ARM::tPOP_RET) {
4686 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands(), N: 2)) {
4687 if (MO.isImplicit() || !MO.isReg())
4688 continue;
4689 Register Reg = MO.getReg();
4690 if (Reg < ARM::R0 || Reg > ARM::R7) {
4691 if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4692 !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4693 ErrInfo = "Unsupported register in Thumb1 push/pop";
4694 return false;
4695 }
4696 }
4697 }
4698 }
4699 if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
4700 assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
4701 if ((MI.getOperand(i: 4).getImm() != 2 && MI.getOperand(i: 4).getImm() != 3) ||
4702 MI.getOperand(i: 4).getImm() != MI.getOperand(i: 5).getImm() + 2) {
4703 ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
4704 return false;
4705 }
4706 }
4707
4708 // Check the address model by taking the first Imm operand and checking it is
4709 // legal for that addressing mode.
4710 ARMII::AddrMode AddrMode =
4711 (ARMII::AddrMode)(MI.getDesc().TSFlags & ARMII::AddrModeMask);
4712 switch (AddrMode) {
4713 default:
4714 break;
4715 case ARMII::AddrModeT2_i7:
4716 case ARMII::AddrModeT2_i7s2:
4717 case ARMII::AddrModeT2_i7s4:
4718 case ARMII::AddrModeT2_i8:
4719 case ARMII::AddrModeT2_i8pos:
4720 case ARMII::AddrModeT2_i8neg:
4721 case ARMII::AddrModeT2_i8s4:
4722 case ARMII::AddrModeT2_i12: {
4723 uint32_t Imm = 0;
4724 for (auto Op : MI.operands()) {
4725 if (Op.isImm()) {
4726 Imm = Op.getImm();
4727 break;
4728 }
4729 }
4730 if (!isLegalAddressImm(Opcode: MI.getOpcode(), Imm, TII: this)) {
4731 ErrInfo = "Incorrect AddrMode Imm for instruction";
4732 return false;
4733 }
4734 break;
4735 }
4736 }
4737 return true;
4738}
4739
4740void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4741 unsigned LoadImmOpc,
4742 unsigned LoadOpc) const {
4743 assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4744 "ROPI/RWPI not currently supported with stack guard");
4745
4746 MachineBasicBlock &MBB = *MI->getParent();
4747 DebugLoc DL = MI->getDebugLoc();
4748 Register Reg = MI->getOperand(i: 0).getReg();
4749 MachineInstrBuilder MIB;
4750 unsigned int Offset = 0;
4751
4752 if (LoadImmOpc == ARM::MRC || LoadImmOpc == ARM::t2MRC) {
4753 assert(!Subtarget.isReadTPSoft() &&
4754 "TLS stack protector requires hardware TLS register");
4755
4756 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4757 .addImm(Val: 15)
4758 .addImm(Val: 0)
4759 .addImm(Val: 13)
4760 .addImm(Val: 0)
4761 .addImm(Val: 3)
4762 .add(MOs: predOps(Pred: ARMCC::AL));
4763
4764 Module &M = *MBB.getParent()->getFunction().getParent();
4765 Offset = M.getStackProtectorGuardOffset();
4766 if (Offset & ~0xfffU) {
4767 // The offset won't fit in the LDR's 12-bit immediate field, so emit an
4768 // extra ADD to cover the delta. This gives us a guaranteed 8 additional
4769 // bits, resulting in a range of 0 to +1 MiB for the guard offset.
4770 unsigned AddOpc = (LoadImmOpc == ARM::MRC) ? ARM::ADDri : ARM::t2ADDri;
4771 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AddOpc), DestReg: Reg)
4772 .addReg(RegNo: Reg, Flags: RegState::Kill)
4773 .addImm(Val: Offset & ~0xfffU)
4774 .add(MOs: predOps(Pred: ARMCC::AL))
4775 .addReg(RegNo: 0);
4776 Offset &= 0xfffU;
4777 }
4778 } else {
4779 const GlobalValue *GV =
4780 cast<GlobalValue>(Val: (*MI->memoperands_begin())->getValue());
4781 bool IsIndirect = Subtarget.isGVIndirectSymbol(GV);
4782
4783 unsigned TargetFlags = ARMII::MO_NO_FLAG;
4784 if (Subtarget.isTargetMachO()) {
4785 TargetFlags |= ARMII::MO_NONLAZY;
4786 } else if (Subtarget.isTargetCOFF()) {
4787 if (GV->hasDLLImportStorageClass())
4788 TargetFlags |= ARMII::MO_DLLIMPORT;
4789 else if (IsIndirect)
4790 TargetFlags |= ARMII::MO_COFFSTUB;
4791 } else if (IsIndirect) {
4792 TargetFlags |= ARMII::MO_GOT;
4793 }
4794
4795 if (LoadImmOpc == ARM::tMOVi32imm) { // Thumb-1 execute-only
4796 Register CPSRSaveReg = ARM::R12; // Use R12 as scratch register
4797 auto APSREncoding =
4798 ARMSysReg::lookupMClassSysRegByName(Name: "apsr_nzcvq")->Encoding;
4799 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ARM::t2MRS_M), DestReg: CPSRSaveReg)
4800 .addImm(Val: APSREncoding)
4801 .add(MOs: predOps(Pred: ARMCC::AL));
4802 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4803 .addGlobalAddress(GV, Offset: 0, TargetFlags);
4804 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ARM::t2MSR_M))
4805 .addImm(Val: APSREncoding)
4806 .addReg(RegNo: CPSRSaveReg, Flags: RegState::Kill)
4807 .add(MOs: predOps(Pred: ARMCC::AL));
4808 } else {
4809 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadImmOpc), DestReg: Reg)
4810 .addGlobalAddress(GV, Offset: 0, TargetFlags);
4811 }
4812
4813 if (IsIndirect) {
4814 MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadOpc), DestReg: Reg);
4815 MIB.addReg(RegNo: Reg, Flags: RegState::Kill).addImm(Val: 0);
4816 auto Flags = MachineMemOperand::MOLoad |
4817 MachineMemOperand::MODereferenceable |
4818 MachineMemOperand::MOInvariant;
4819 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4820 PtrInfo: MachinePointerInfo::getGOT(MF&: *MBB.getParent()), F: Flags, Size: 4, BaseAlignment: Align(4));
4821 MIB.addMemOperand(MMO).add(MOs: predOps(Pred: ARMCC::AL));
4822 }
4823 }
4824
4825 MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: LoadOpc), DestReg: Reg);
4826 MIB.addReg(RegNo: Reg, Flags: RegState::Kill)
4827 .addImm(Val: Offset)
4828 .cloneMemRefs(OtherMI: *MI)
4829 .add(MOs: predOps(Pred: ARMCC::AL));
4830}
4831
4832bool
4833ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
4834 unsigned &AddSubOpc,
4835 bool &NegAcc, bool &HasLane) const {
4836 auto I = MLxEntryMap.find(Val: Opcode);
4837 if (I == MLxEntryMap.end())
4838 return false;
4839
4840 const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
4841 MulOpc = Entry.MulOpc;
4842 AddSubOpc = Entry.AddSubOpc;
4843 NegAcc = Entry.NegAcc;
4844 HasLane = Entry.HasLane;
4845 return true;
4846}
4847
4848//===----------------------------------------------------------------------===//
4849// Execution domains.
4850//===----------------------------------------------------------------------===//
4851//
4852// Some instructions go down the NEON pipeline, some go down the VFP pipeline,
4853// and some can go down both. The vmov instructions go down the VFP pipeline,
4854// but they can be changed to vorr equivalents that are executed by the NEON
4855// pipeline.
4856//
4857// We use the following execution domain numbering:
4858//
4859enum ARMExeDomain {
4860 ExeGeneric = 0,
4861 ExeVFP = 1,
4862 ExeNEON = 2
4863};
4864
4865//
4866// Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
4867//
4868std::pair<uint16_t, uint16_t>
4869ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
4870 // If we don't have access to NEON instructions then we won't be able
4871 // to swizzle anything to the NEON domain. Check to make sure.
4872 if (Subtarget.hasNEON()) {
4873 // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
4874 // if they are not predicated.
4875 if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
4876 return std::make_pair(x: ExeVFP, y: (1 << ExeVFP) | (1 << ExeNEON));
4877
4878 // CortexA9 is particularly picky about mixing the two and wants these
4879 // converted.
4880 if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
4881 (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
4882 MI.getOpcode() == ARM::VMOVS))
4883 return std::make_pair(x: ExeVFP, y: (1 << ExeVFP) | (1 << ExeNEON));
4884 }
4885 // No other instructions can be swizzled, so just determine their domain.
4886 unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
4887
4888 if (Domain & ARMII::DomainNEON)
4889 return std::make_pair(x: ExeNEON, y: 0);
4890
4891 // Certain instructions can go either way on Cortex-A8.
4892 // Treat them as NEON instructions.
4893 if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
4894 return std::make_pair(x: ExeNEON, y: 0);
4895
4896 if (Domain & ARMII::DomainVFP)
4897 return std::make_pair(x: ExeVFP, y: 0);
4898
4899 return std::make_pair(x: ExeGeneric, y: 0);
4900}
4901
4902static MCRegister getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
4903 unsigned SReg, unsigned &Lane) {
4904 MCRegister DReg =
4905 TRI->getMatchingSuperReg(Reg: SReg, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
4906 Lane = 0;
4907
4908 if (DReg)
4909 return DReg;
4910
4911 Lane = 1;
4912 DReg = TRI->getMatchingSuperReg(Reg: SReg, SubIdx: ARM::ssub_1, RC: &ARM::DPRRegClass);
4913
4914 assert(DReg && "S-register with no D super-register?");
4915 return DReg;
4916}
4917
4918/// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
4919/// set ImplicitSReg to a register number that must be marked as implicit-use or
4920/// zero if no register needs to be defined as implicit-use.
4921///
4922/// If the function cannot determine if an SPR should be marked implicit use or
4923/// not, it returns false.
4924///
4925/// This function handles cases where an instruction is being modified from taking
4926/// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
4927/// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
4928/// lane of the DPR).
4929///
4930/// If the other SPR is defined, an implicit-use of it should be added. Else,
4931/// (including the case where the DPR itself is defined), it should not.
4932///
4933static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
4934 MachineInstr &MI, MCRegister DReg,
4935 unsigned Lane,
4936 MCRegister &ImplicitSReg) {
4937 // If the DPR is defined or used already, the other SPR lane will be chained
4938 // correctly, so there is nothing to be done.
4939 if (MI.definesRegister(Reg: DReg, TRI) || MI.readsRegister(Reg: DReg, TRI)) {
4940 ImplicitSReg = MCRegister();
4941 return true;
4942 }
4943
4944 // Otherwise we need to go searching to see if the SPR is set explicitly.
4945 ImplicitSReg = TRI->getSubReg(Reg: DReg,
4946 Idx: (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
4947 MachineBasicBlock::LivenessQueryResult LQR =
4948 MI.getParent()->computeRegisterLiveness(TRI, Reg: ImplicitSReg, Before: MI);
4949
4950 if (LQR == MachineBasicBlock::LQR_Live)
4951 return true;
4952 else if (LQR == MachineBasicBlock::LQR_Unknown)
4953 return false;
4954
4955 // If the register is known not to be live, there is no need to add an
4956 // implicit-use.
4957 ImplicitSReg = MCRegister();
4958 return true;
4959}
4960
4961void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
4962 unsigned Domain) const {
4963 unsigned DstReg, SrcReg;
4964 MCRegister DReg;
4965 unsigned Lane;
4966 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
4967 const TargetRegisterInfo *TRI = &getRegisterInfo();
4968 switch (MI.getOpcode()) {
4969 default:
4970 llvm_unreachable("cannot handle opcode!");
4971 break;
4972 case ARM::VMOVD:
4973 if (Domain != ExeNEON)
4974 break;
4975
4976 // Zap the predicate operands.
4977 assert(!isPredicated(MI) && "Cannot predicate a VORRd");
4978
4979 // Make sure we've got NEON instructions.
4980 assert(Subtarget.hasNEON() && "VORRd requires NEON");
4981
4982 // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
4983 DstReg = MI.getOperand(i: 0).getReg();
4984 SrcReg = MI.getOperand(i: 1).getReg();
4985
4986 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
4987 MI.removeOperand(OpNo: i - 1);
4988
4989 // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
4990 MI.setDesc(get(Opcode: ARM::VORRd));
4991 MIB.addReg(RegNo: DstReg, Flags: RegState::Define)
4992 .addReg(RegNo: SrcReg)
4993 .addReg(RegNo: SrcReg)
4994 .add(MOs: predOps(Pred: ARMCC::AL));
4995 break;
4996 case ARM::VMOVRS:
4997 if (Domain != ExeNEON)
4998 break;
4999 assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
5000
5001 // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
5002 DstReg = MI.getOperand(i: 0).getReg();
5003 SrcReg = MI.getOperand(i: 1).getReg();
5004
5005 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5006 MI.removeOperand(OpNo: i - 1);
5007
5008 DReg = getCorrespondingDRegAndLane(TRI, SReg: SrcReg, Lane);
5009
5010 // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5011 // Note that DSrc has been widened and the other lane may be undef, which
5012 // contaminates the entire register.
5013 MI.setDesc(get(Opcode: ARM::VGETLNi32));
5014 MIB.addReg(RegNo: DstReg, Flags: RegState::Define)
5015 .addReg(RegNo: DReg, Flags: RegState::Undef)
5016 .addImm(Val: Lane)
5017 .add(MOs: predOps(Pred: ARMCC::AL));
5018
5019 // The old source should be an implicit use, otherwise we might think it
5020 // was dead before here.
5021 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5022 break;
5023 case ARM::VMOVSR: {
5024 if (Domain != ExeNEON)
5025 break;
5026 assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5027
5028 // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5029 DstReg = MI.getOperand(i: 0).getReg();
5030 SrcReg = MI.getOperand(i: 1).getReg();
5031
5032 DReg = getCorrespondingDRegAndLane(TRI, SReg: DstReg, Lane);
5033
5034 MCRegister ImplicitSReg;
5035 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5036 break;
5037
5038 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5039 MI.removeOperand(OpNo: i - 1);
5040
5041 // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5042 // Again DDst may be undefined at the beginning of this instruction.
5043 MI.setDesc(get(Opcode: ARM::VSETLNi32));
5044 MIB.addReg(RegNo: DReg, Flags: RegState::Define)
5045 .addReg(RegNo: DReg, Flags: getUndefRegState(B: !MI.readsRegister(Reg: DReg, TRI)))
5046 .addReg(RegNo: SrcReg)
5047 .addImm(Val: Lane)
5048 .add(MOs: predOps(Pred: ARMCC::AL));
5049
5050 // The narrower destination must be marked as set to keep previous chains
5051 // in place.
5052 MIB.addReg(RegNo: DstReg, Flags: RegState::Define | RegState::Implicit);
5053 if (ImplicitSReg)
5054 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5055 break;
5056 }
5057 case ARM::VMOVS: {
5058 if (Domain != ExeNEON)
5059 break;
5060
5061 // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5062 DstReg = MI.getOperand(i: 0).getReg();
5063 SrcReg = MI.getOperand(i: 1).getReg();
5064
5065 unsigned DstLane = 0, SrcLane = 0;
5066 MCRegister DDst, DSrc;
5067 DDst = getCorrespondingDRegAndLane(TRI, SReg: DstReg, Lane&: DstLane);
5068 DSrc = getCorrespondingDRegAndLane(TRI, SReg: SrcReg, Lane&: SrcLane);
5069
5070 MCRegister ImplicitSReg;
5071 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg: DSrc, Lane: SrcLane, ImplicitSReg))
5072 break;
5073
5074 for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5075 MI.removeOperand(OpNo: i - 1);
5076
5077 if (DSrc == DDst) {
5078 // Destination can be:
5079 // %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5080 MI.setDesc(get(Opcode: ARM::VDUPLN32d));
5081 MIB.addReg(RegNo: DDst, Flags: RegState::Define)
5082 .addReg(RegNo: DDst, Flags: getUndefRegState(B: !MI.readsRegister(Reg: DDst, TRI)))
5083 .addImm(Val: SrcLane)
5084 .add(MOs: predOps(Pred: ARMCC::AL));
5085
5086 // Neither the source or the destination are naturally represented any
5087 // more, so add them in manually.
5088 MIB.addReg(RegNo: DstReg, Flags: RegState::Implicit | RegState::Define);
5089 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5090 if (ImplicitSReg)
5091 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5092 break;
5093 }
5094
5095 // In general there's no single instruction that can perform an S <-> S
5096 // move in NEON space, but a pair of VEXT instructions *can* do the
5097 // job. It turns out that the VEXTs needed will only use DSrc once, with
5098 // the position based purely on the combination of lane-0 and lane-1
5099 // involved. For example
5100 // vmov s0, s2 -> vext.32 d0, d0, d1, #1 vext.32 d0, d0, d0, #1
5101 // vmov s1, s3 -> vext.32 d0, d1, d0, #1 vext.32 d0, d0, d0, #1
5102 // vmov s0, s3 -> vext.32 d0, d0, d0, #1 vext.32 d0, d1, d0, #1
5103 // vmov s1, s2 -> vext.32 d0, d0, d0, #1 vext.32 d0, d0, d1, #1
5104 //
5105 // Pattern of the MachineInstrs is:
5106 // %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5107 MachineInstrBuilder NewMIB;
5108 NewMIB = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: ARM::VEXTd32),
5109 DestReg: DDst);
5110
5111 // On the first instruction, both DSrc and DDst may be undef if present.
5112 // Specifically when the original instruction didn't have them as an
5113 // <imp-use>.
5114 MCRegister CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5115 bool CurUndef = !MI.readsRegister(Reg: CurReg, TRI);
5116 NewMIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef));
5117
5118 CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5119 CurUndef = !MI.readsRegister(Reg: CurReg, TRI);
5120 NewMIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef))
5121 .addImm(Val: 1)
5122 .add(MOs: predOps(Pred: ARMCC::AL));
5123
5124 if (SrcLane == DstLane)
5125 NewMIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5126
5127 MI.setDesc(get(Opcode: ARM::VEXTd32));
5128 MIB.addReg(RegNo: DDst, Flags: RegState::Define);
5129
5130 // On the second instruction, DDst has definitely been defined above, so
5131 // it is not undef. DSrc, if present, can be undef as above.
5132 CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5133 CurUndef = CurReg == DSrc && !MI.readsRegister(Reg: CurReg, TRI);
5134 MIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef));
5135
5136 CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5137 CurUndef = CurReg == DSrc && !MI.readsRegister(Reg: CurReg, TRI);
5138 MIB.addReg(RegNo: CurReg, Flags: getUndefRegState(B: CurUndef))
5139 .addImm(Val: 1)
5140 .add(MOs: predOps(Pred: ARMCC::AL));
5141
5142 if (SrcLane != DstLane)
5143 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5144
5145 // As before, the original destination is no longer represented, add it
5146 // implicitly.
5147 MIB.addReg(RegNo: DstReg, Flags: RegState::Define | RegState::Implicit);
5148 if (ImplicitSReg != 0)
5149 MIB.addReg(RegNo: ImplicitSReg, Flags: RegState::Implicit);
5150 break;
5151 }
5152 }
5153}
5154
5155//===----------------------------------------------------------------------===//
5156// Partial register updates
5157//===----------------------------------------------------------------------===//
5158//
5159// Swift renames NEON registers with 64-bit granularity. That means any
5160// instruction writing an S-reg implicitly reads the containing D-reg. The
5161// problem is mostly avoided by translating f32 operations to v2f32 operations
5162// on D-registers, but f32 loads are still a problem.
5163//
5164// These instructions can load an f32 into a NEON register:
5165//
5166// VLDRS - Only writes S, partial D update.
5167// VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5168// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5169//
5170// FCONSTD can be used as a dependency-breaking instruction.
5171unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5172 const MachineInstr &MI, unsigned OpNum,
5173 const TargetRegisterInfo *TRI) const {
5174 auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5175 if (!PartialUpdateClearance)
5176 return 0;
5177
5178 assert(TRI && "Need TRI instance");
5179
5180 const MachineOperand &MO = MI.getOperand(i: OpNum);
5181 if (MO.readsReg())
5182 return 0;
5183 Register Reg = MO.getReg();
5184 int UseOp = -1;
5185
5186 switch (MI.getOpcode()) {
5187 // Normal instructions writing only an S-register.
5188 case ARM::VLDRS:
5189 case ARM::FCONSTS:
5190 case ARM::VMOVSR:
5191 case ARM::VMOVv8i8:
5192 case ARM::VMOVv4i16:
5193 case ARM::VMOVv2i32:
5194 case ARM::VMOVv2f32:
5195 case ARM::VMOVv1i64:
5196 UseOp = MI.findRegisterUseOperandIdx(Reg, TRI, isKill: false);
5197 break;
5198
5199 // Explicitly reads the dependency.
5200 case ARM::VLD1LNd32:
5201 UseOp = 3;
5202 break;
5203 default:
5204 return 0;
5205 }
5206
5207 // If this instruction actually reads a value from Reg, there is no unwanted
5208 // dependency.
5209 if (UseOp != -1 && MI.getOperand(i: UseOp).readsReg())
5210 return 0;
5211
5212 // We must be able to clobber the whole D-reg.
5213 if (Reg.isVirtual()) {
5214 // Virtual register must be a def undef foo:ssub_0 operand.
5215 if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5216 return 0;
5217 } else if (ARM::SPRRegClass.contains(Reg)) {
5218 // Physical register: MI must define the full D-reg.
5219 MCRegister DReg =
5220 TRI->getMatchingSuperReg(Reg, SubIdx: ARM::ssub_0, RC: &ARM::DPRRegClass);
5221 if (!DReg || !MI.definesRegister(Reg: DReg, TRI))
5222 return 0;
5223 }
5224
5225 // MI has an unwanted D-register dependency.
5226 // Avoid defs in the previous N instructrions.
5227 return PartialUpdateClearance;
5228}
5229
5230// Break a partial register dependency after getPartialRegUpdateClearance
5231// returned non-zero.
5232void ARMBaseInstrInfo::breakPartialRegDependency(
5233 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5234 assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5235 assert(TRI && "Need TRI instance");
5236
5237 const MachineOperand &MO = MI.getOperand(i: OpNum);
5238 Register Reg = MO.getReg();
5239 assert(Reg.isPhysical() && "Can't break virtual register dependencies.");
5240 unsigned DReg = Reg;
5241
5242 // If MI defines an S-reg, find the corresponding D super-register.
5243 if (ARM::SPRRegClass.contains(Reg)) {
5244 DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5245 assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5246 }
5247
5248 assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5249 assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5250
5251 // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5252 // the full D-register by loading the same value to both lanes. The
5253 // instruction is micro-coded with 2 uops, so don't do this until we can
5254 // properly schedule micro-coded instructions. The dispatcher stalls cause
5255 // too big regressions.
5256
5257 // Insert the dependency-breaking FCONSTD before MI.
5258 // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5259 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: ARM::FCONSTD), DestReg: DReg)
5260 .addImm(Val: 96)
5261 .add(MOs: predOps(Pred: ARMCC::AL));
5262 MI.addRegisterKilled(IncomingReg: DReg, RegInfo: TRI, AddIfNotFound: true);
5263}
5264
5265bool ARMBaseInstrInfo::hasNOP() const {
5266 return Subtarget.hasFeature(Feature: ARM::HasV6KOps);
5267}
5268
5269bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5270 if (MI->getNumOperands() < 4)
5271 return true;
5272 unsigned ShOpVal = MI->getOperand(i: 3).getImm();
5273 unsigned ShImm = ARM_AM::getSORegOffset(Op: ShOpVal);
5274 // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5275 if ((ShImm == 1 && ARM_AM::getSORegShOp(Op: ShOpVal) == ARM_AM::lsr) ||
5276 ((ShImm == 1 || ShImm == 2) &&
5277 ARM_AM::getSORegShOp(Op: ShOpVal) == ARM_AM::lsl))
5278 return true;
5279
5280 return false;
5281}
5282
5283bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5284 const MachineInstr &MI, unsigned DefIdx,
5285 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5286 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5287 assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5288
5289 switch (MI.getOpcode()) {
5290 case ARM::VMOVDRR:
5291 // dX = VMOVDRR rY, rZ
5292 // is the same as:
5293 // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5294 // Populate the InputRegs accordingly.
5295 // rY
5296 const MachineOperand *MOReg = &MI.getOperand(i: 1);
5297 if (!MOReg->isUndef())
5298 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg->getReg(),
5299 MOReg->getSubReg(), ARM::ssub_0));
5300 // rZ
5301 MOReg = &MI.getOperand(i: 2);
5302 if (!MOReg->isUndef())
5303 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg->getReg(),
5304 MOReg->getSubReg(), ARM::ssub_1));
5305 return true;
5306 }
5307 llvm_unreachable("Target dependent opcode missing");
5308}
5309
5310bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5311 const MachineInstr &MI, unsigned DefIdx,
5312 RegSubRegPairAndIdx &InputReg) const {
5313 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5314 assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5315
5316 switch (MI.getOpcode()) {
5317 case ARM::VMOVRRD:
5318 // rX, rY = VMOVRRD dZ
5319 // is the same as:
5320 // rX = EXTRACT_SUBREG dZ, ssub_0
5321 // rY = EXTRACT_SUBREG dZ, ssub_1
5322 const MachineOperand &MOReg = MI.getOperand(i: 2);
5323 if (MOReg.isUndef())
5324 return false;
5325 InputReg.Reg = MOReg.getReg();
5326 InputReg.SubReg = MOReg.getSubReg();
5327 InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5328 return true;
5329 }
5330 llvm_unreachable("Target dependent opcode missing");
5331}
5332
5333bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5334 const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5335 RegSubRegPairAndIdx &InsertedReg) const {
5336 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5337 assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5338
5339 switch (MI.getOpcode()) {
5340 case ARM::VSETLNi32:
5341 case ARM::MVE_VMOV_to_lane_32:
5342 // dX = VSETLNi32 dY, rZ, imm
5343 // qX = MVE_VMOV_to_lane_32 qY, rZ, imm
5344 const MachineOperand &MOBaseReg = MI.getOperand(i: 1);
5345 const MachineOperand &MOInsertedReg = MI.getOperand(i: 2);
5346 if (MOInsertedReg.isUndef())
5347 return false;
5348 const MachineOperand &MOIndex = MI.getOperand(i: 3);
5349 BaseReg.Reg = MOBaseReg.getReg();
5350 BaseReg.SubReg = MOBaseReg.getSubReg();
5351
5352 InsertedReg.Reg = MOInsertedReg.getReg();
5353 InsertedReg.SubReg = MOInsertedReg.getSubReg();
5354 InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm();
5355 return true;
5356 }
5357 llvm_unreachable("Target dependent opcode missing");
5358}
5359
5360std::pair<unsigned, unsigned>
5361ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5362 const unsigned Mask = ARMII::MO_OPTION_MASK;
5363 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
5364}
5365
5366ArrayRef<std::pair<unsigned, const char *>>
5367ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5368 using namespace ARMII;
5369
5370 static const std::pair<unsigned, const char *> TargetFlags[] = {
5371 {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"},
5372 {MO_LO_0_7, "arm-lo-0-7"}, {MO_HI_0_7, "arm-hi-0-7"},
5373 {MO_LO_8_15, "arm-lo-8-15"}, {MO_HI_8_15, "arm-hi-8-15"},
5374 };
5375 return ArrayRef(TargetFlags);
5376}
5377
5378ArrayRef<std::pair<unsigned, const char *>>
5379ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5380 using namespace ARMII;
5381
5382 static const std::pair<unsigned, const char *> TargetFlags[] = {
5383 {MO_COFFSTUB, "arm-coffstub"},
5384 {MO_GOT, "arm-got"},
5385 {MO_SBREL, "arm-sbrel"},
5386 {MO_DLLIMPORT, "arm-dllimport"},
5387 {MO_SECREL, "arm-secrel"},
5388 {MO_NONLAZY, "arm-nonlazy"}};
5389 return ArrayRef(TargetFlags);
5390}
5391
5392std::optional<RegImmPair>
5393ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
5394 int Sign = 1;
5395 unsigned Opcode = MI.getOpcode();
5396 int64_t Offset = 0;
5397
5398 // TODO: Handle cases where Reg is a super- or sub-register of the
5399 // destination register.
5400 const MachineOperand &Op0 = MI.getOperand(i: 0);
5401 if (!Op0.isReg() || Reg != Op0.getReg())
5402 return std::nullopt;
5403
5404 // We describe SUBri or ADDri instructions.
5405 if (Opcode == ARM::SUBri)
5406 Sign = -1;
5407 else if (Opcode != ARM::ADDri)
5408 return std::nullopt;
5409
5410 // TODO: Third operand can be global address (usually some string). Since
5411 // strings can be relocated we cannot calculate their offsets for
5412 // now.
5413 if (!MI.getOperand(i: 1).isReg() || !MI.getOperand(i: 2).isImm())
5414 return std::nullopt;
5415
5416 Offset = MI.getOperand(i: 2).getImm() * Sign;
5417 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
5418}
5419
5420bool llvm::registerDefinedBetween(unsigned Reg,
5421 MachineBasicBlock::iterator From,
5422 MachineBasicBlock::iterator To,
5423 const TargetRegisterInfo *TRI) {
5424 for (auto I = From; I != To; ++I)
5425 if (I->modifiesRegister(Reg, TRI))
5426 return true;
5427 return false;
5428}
5429
5430MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5431 const TargetRegisterInfo *TRI) {
5432 // Search backwards to the instruction that defines CSPR. This may or not
5433 // be a CMP, we check that after this loop. If we find another instruction
5434 // that reads cpsr, we return nullptr.
5435 MachineBasicBlock::iterator CmpMI = Br;
5436 while (CmpMI != Br->getParent()->begin()) {
5437 --CmpMI;
5438 if (CmpMI->modifiesRegister(Reg: ARM::CPSR, TRI))
5439 break;
5440 if (CmpMI->readsRegister(Reg: ARM::CPSR, TRI))
5441 break;
5442 }
5443
5444 // Check that this inst is a CMP r[0-7], #0 and that the register
5445 // is not redefined between the cmp and the br.
5446 if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5447 return nullptr;
5448 Register Reg = CmpMI->getOperand(i: 0).getReg();
5449 Register PredReg;
5450 ARMCC::CondCodes Pred = getInstrPredicate(MI: *CmpMI, PredReg);
5451 if (Pred != ARMCC::AL || CmpMI->getOperand(i: 1).getImm() != 0)
5452 return nullptr;
5453 if (!isARMLowRegister(Reg))
5454 return nullptr;
5455 if (registerDefinedBetween(Reg, From: CmpMI->getNextNode(), To: Br, TRI))
5456 return nullptr;
5457
5458 return &*CmpMI;
5459}
5460
5461unsigned llvm::ConstantMaterializationCost(unsigned Val,
5462 const ARMSubtarget *Subtarget,
5463 bool ForCodesize) {
5464 if (Subtarget->isThumb()) {
5465 if (Val <= 255) // MOV
5466 return ForCodesize ? 2 : 1;
5467 if (Subtarget->hasV6T2Ops() && (Val <= 0xffff || // MOV
5468 ARM_AM::getT2SOImmVal(Arg: Val) != -1 || // MOVW
5469 ARM_AM::getT2SOImmVal(Arg: ~Val) != -1)) // MVN
5470 return ForCodesize ? 4 : 1;
5471 if (Val <= 510) // MOV + ADDi8
5472 return ForCodesize ? 4 : 2;
5473 if (~Val <= 255) // MOV + MVN
5474 return ForCodesize ? 4 : 2;
5475 if (ARM_AM::isThumbImmShiftedVal(V: Val)) // MOV + LSL
5476 return ForCodesize ? 4 : 2;
5477 } else {
5478 if (ARM_AM::getSOImmVal(Arg: Val) != -1) // MOV
5479 return ForCodesize ? 4 : 1;
5480 if (ARM_AM::getSOImmVal(Arg: ~Val) != -1) // MVN
5481 return ForCodesize ? 4 : 1;
5482 if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5483 return ForCodesize ? 4 : 1;
5484 if (ARM_AM::isSOImmTwoPartVal(V: Val)) // two instrs
5485 return ForCodesize ? 8 : 2;
5486 if (ARM_AM::isSOImmTwoPartValNeg(V: Val)) // two instrs
5487 return ForCodesize ? 8 : 2;
5488 }
5489 if (Subtarget->useMovt()) // MOVW + MOVT
5490 return ForCodesize ? 8 : 2;
5491 return ForCodesize ? 8 : 3; // Literal pool load
5492}
5493
5494bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5495 const ARMSubtarget *Subtarget,
5496 bool ForCodesize) {
5497 // Check with ForCodesize
5498 unsigned Cost1 = ConstantMaterializationCost(Val: Val1, Subtarget, ForCodesize);
5499 unsigned Cost2 = ConstantMaterializationCost(Val: Val2, Subtarget, ForCodesize);
5500 if (Cost1 < Cost2)
5501 return true;
5502 if (Cost1 > Cost2)
5503 return false;
5504
5505 // If they are equal, try with !ForCodesize
5506 return ConstantMaterializationCost(Val: Val1, Subtarget, ForCodesize: !ForCodesize) <
5507 ConstantMaterializationCost(Val: Val2, Subtarget, ForCodesize: !ForCodesize);
5508}
5509
5510/// Constants defining how certain sequences should be outlined.
5511/// This encompasses how an outlined function should be called, and what kind of
5512/// frame should be emitted for that outlined function.
5513///
5514/// \p MachineOutlinerTailCall implies that the function is being created from
5515/// a sequence of instructions ending in a return.
5516///
5517/// That is,
5518///
5519/// I1 OUTLINED_FUNCTION:
5520/// I2 --> B OUTLINED_FUNCTION I1
5521/// BX LR I2
5522/// BX LR
5523///
5524/// +-------------------------+--------+-----+
5525/// | | Thumb2 | ARM |
5526/// +-------------------------+--------+-----+
5527/// | Call overhead in Bytes | 4 | 4 |
5528/// | Frame overhead in Bytes | 0 | 0 |
5529/// | Stack fixup required | No | No |
5530/// +-------------------------+--------+-----+
5531///
5532/// \p MachineOutlinerThunk implies that the function is being created from
5533/// a sequence of instructions ending in a call. The outlined function is
5534/// called with a BL instruction, and the outlined function tail-calls the
5535/// original call destination.
5536///
5537/// That is,
5538///
5539/// I1 OUTLINED_FUNCTION:
5540/// I2 --> BL OUTLINED_FUNCTION I1
5541/// BL f I2
5542/// B f
5543///
5544/// +-------------------------+--------+-----+
5545/// | | Thumb2 | ARM |
5546/// +-------------------------+--------+-----+
5547/// | Call overhead in Bytes | 4 | 4 |
5548/// | Frame overhead in Bytes | 0 | 0 |
5549/// | Stack fixup required | No | No |
5550/// +-------------------------+--------+-----+
5551///
5552/// \p MachineOutlinerNoLRSave implies that the function should be called using
5553/// a BL instruction, but doesn't require LR to be saved and restored. This
5554/// happens when LR is known to be dead.
5555///
5556/// That is,
5557///
5558/// I1 OUTLINED_FUNCTION:
5559/// I2 --> BL OUTLINED_FUNCTION I1
5560/// I3 I2
5561/// I3
5562/// BX LR
5563///
5564/// +-------------------------+--------+-----+
5565/// | | Thumb2 | ARM |
5566/// +-------------------------+--------+-----+
5567/// | Call overhead in Bytes | 4 | 4 |
5568/// | Frame overhead in Bytes | 2 | 4 |
5569/// | Stack fixup required | No | No |
5570/// +-------------------------+--------+-----+
5571///
5572/// \p MachineOutlinerRegSave implies that the function should be called with a
5573/// save and restore of LR to an available register. This allows us to avoid
5574/// stack fixups. Note that this outlining variant is compatible with the
5575/// NoLRSave case.
5576///
5577/// That is,
5578///
5579/// I1 Save LR OUTLINED_FUNCTION:
5580/// I2 --> BL OUTLINED_FUNCTION I1
5581/// I3 Restore LR I2
5582/// I3
5583/// BX LR
5584///
5585/// +-------------------------+--------+-----+
5586/// | | Thumb2 | ARM |
5587/// +-------------------------+--------+-----+
5588/// | Call overhead in Bytes | 8 | 12 |
5589/// | Frame overhead in Bytes | 2 | 4 |
5590/// | Stack fixup required | No | No |
5591/// +-------------------------+--------+-----+
5592///
5593/// \p MachineOutlinerDefault implies that the function should be called with
5594/// a save and restore of LR to the stack.
5595///
5596/// That is,
5597///
5598/// I1 Save LR OUTLINED_FUNCTION:
5599/// I2 --> BL OUTLINED_FUNCTION I1
5600/// I3 Restore LR I2
5601/// I3
5602/// BX LR
5603///
5604/// +-------------------------+--------+-----+
5605/// | | Thumb2 | ARM |
5606/// +-------------------------+--------+-----+
5607/// | Call overhead in Bytes | 8 | 12 |
5608/// | Frame overhead in Bytes | 2 | 4 |
5609/// | Stack fixup required | Yes | Yes |
5610/// +-------------------------+--------+-----+
5611
5612enum MachineOutlinerClass {
5613 MachineOutlinerTailCall,
5614 MachineOutlinerThunk,
5615 MachineOutlinerNoLRSave,
5616 MachineOutlinerRegSave,
5617 MachineOutlinerDefault
5618};
5619
5620enum MachineOutlinerMBBFlags {
5621 LRUnavailableSomewhere = 0x2,
5622 HasCalls = 0x4,
5623 UnsafeRegsDead = 0x8
5624};
5625
5626struct OutlinerCosts {
5627 int CallTailCall;
5628 int FrameTailCall;
5629 int CallThunk;
5630 int FrameThunk;
5631 int CallNoLRSave;
5632 int FrameNoLRSave;
5633 int CallRegSave;
5634 int FrameRegSave;
5635 int CallDefault;
5636 int FrameDefault;
5637 int SaveRestoreLROnStack;
5638
5639 OutlinerCosts(const ARMSubtarget &target)
5640 : CallTailCall(target.isThumb() ? 4 : 4),
5641 FrameTailCall(target.isThumb() ? 0 : 0),
5642 CallThunk(target.isThumb() ? 4 : 4),
5643 FrameThunk(target.isThumb() ? 0 : 0),
5644 CallNoLRSave(target.isThumb() ? 4 : 4),
5645 FrameNoLRSave(target.isThumb() ? 2 : 4),
5646 CallRegSave(target.isThumb() ? 8 : 12),
5647 FrameRegSave(target.isThumb() ? 2 : 4),
5648 CallDefault(target.isThumb() ? 8 : 12),
5649 FrameDefault(target.isThumb() ? 2 : 4),
5650 SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {}
5651};
5652
5653Register
5654ARMBaseInstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
5655 MachineFunction *MF = C.getMF();
5656 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
5657 const ARMBaseRegisterInfo *ARI =
5658 static_cast<const ARMBaseRegisterInfo *>(&TRI);
5659
5660 BitVector regsReserved = ARI->getReservedRegs(MF: *MF);
5661 // Check if there is an available register across the sequence that we can
5662 // use.
5663 for (Register Reg : ARM::rGPRRegClass) {
5664 if (!(Reg < regsReserved.size() && regsReserved.test(Idx: Reg)) &&
5665 Reg != ARM::LR && // LR is not reserved, but don't use it.
5666 Reg != ARM::R12 && // R12 is not guaranteed to be preserved.
5667 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
5668 C.isAvailableInsideSeq(Reg, TRI))
5669 return Reg;
5670 }
5671 return Register();
5672}
5673
5674// Compute liveness of LR at the point after the interval [I, E), which
5675// denotes a *backward* iteration through instructions. Used only for return
5676// basic blocks, which do not end with a tail call.
5677static bool isLRAvailable(const TargetRegisterInfo &TRI,
5678 MachineBasicBlock::reverse_iterator I,
5679 MachineBasicBlock::reverse_iterator E) {
5680 // At the end of the function LR dead.
5681 bool Live = false;
5682 for (; I != E; ++I) {
5683 const MachineInstr &MI = *I;
5684
5685 // Check defs of LR.
5686 if (MI.modifiesRegister(Reg: ARM::LR, TRI: &TRI))
5687 Live = false;
5688
5689 // Check uses of LR.
5690 unsigned Opcode = MI.getOpcode();
5691 if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR ||
5692 Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET ||
5693 Opcode == ARM::tBXNS_RET || Opcode == ARM::t2BXAUT_RET) {
5694 // These instructions use LR, but it's not an (explicit or implicit)
5695 // operand.
5696 Live = true;
5697 continue;
5698 }
5699 if (MI.readsRegister(Reg: ARM::LR, TRI: &TRI))
5700 Live = true;
5701 }
5702 return !Live;
5703}
5704
5705/// Return true if \p MI is a call instruction that the outliner can rewrite as
5706/// a tail call.
5707static bool CanTransformInstrIntoTailCall(const MachineInstr &MI) {
5708 auto Opcode = MI.getOpcode();
5709 return (Opcode == ARM::BL || Opcode == ARM::BLX || Opcode == ARM::BLX_noip ||
5710 Opcode == ARM::tBL || Opcode == ARM::tBLXi || Opcode == ARM::tBLXr ||
5711 Opcode == ARM::tBLXr_noip);
5712}
5713
5714std::optional<std::unique_ptr<outliner::OutlinedFunction>>
5715ARMBaseInstrInfo::getOutliningCandidateInfo(
5716 const MachineModuleInfo &MMI,
5717 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
5718 unsigned MinRepeats) const {
5719 unsigned SequenceSize = 0;
5720 for (auto &MI : RepeatedSequenceLocs[0])
5721 SequenceSize += getInstSizeInBytes(MI);
5722
5723 // Properties about candidate MBBs that hold for all of them.
5724 unsigned FlagsSetInAll = 0xF;
5725
5726 // Compute liveness information for each candidate, and set FlagsSetInAll.
5727 const TargetRegisterInfo &TRI = getRegisterInfo();
5728 for (outliner::Candidate &C : RepeatedSequenceLocs)
5729 FlagsSetInAll &= C.Flags;
5730
5731 // According to the ARM Procedure Call Standard, the following are
5732 // undefined on entry/exit from a function call:
5733 //
5734 // * Register R12(IP),
5735 // * Condition codes (and thus the CPSR register)
5736 //
5737 // Since we control the instructions which are part of the outlined regions
5738 // we don't need to be fully compliant with the AAPCS, but we have to
5739 // guarantee that if a veneer is inserted at link time the code is still
5740 // correct. Because of this, we can't outline any sequence of instructions
5741 // where one of these registers is live into/across it. Thus, we need to
5742 // delete those candidates.
5743 auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5744 // If the unsafe registers in this block are all dead, then we don't need
5745 // to compute liveness here.
5746 if (C.Flags & UnsafeRegsDead)
5747 return false;
5748 return C.isAnyUnavailableAcrossOrOutOfSeq(Regs: {ARM::R12, ARM::CPSR}, TRI);
5749 };
5750
5751 // Are there any candidates where those registers are live?
5752 if (!(FlagsSetInAll & UnsafeRegsDead)) {
5753 // Erase every candidate that violates the restrictions above. (It could be
5754 // true that we have viable candidates, so it's not worth bailing out in
5755 // the case that, say, 1 out of 20 candidates violate the restructions.)
5756 llvm::erase_if(C&: RepeatedSequenceLocs, P: CantGuaranteeValueAcrossCall);
5757
5758 // If the sequence doesn't have enough candidates left, then we're done.
5759 if (RepeatedSequenceLocs.size() < MinRepeats)
5760 return std::nullopt;
5761 }
5762
5763 // We expect the majority of the outlining candidates to be in consensus with
5764 // regard to return address sign and authentication, and branch target
5765 // enforcement, in other words, partitioning according to all the four
5766 // possible combinations of PAC-RET and BTI is going to yield one big subset
5767 // and three small (likely empty) subsets. That allows us to cull incompatible
5768 // candidates separately for PAC-RET and BTI.
5769
5770 // Partition the candidates in two sets: one with BTI enabled and one with BTI
5771 // disabled. Remove the candidates from the smaller set. If they are the same
5772 // number prefer the non-BTI ones for outlining, since they have less
5773 // overhead.
5774 auto NoBTI =
5775 llvm::partition(Range&: RepeatedSequenceLocs, P: [](const outliner::Candidate &C) {
5776 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5777 return AFI.branchTargetEnforcement();
5778 });
5779 if (std::distance(first: RepeatedSequenceLocs.begin(), last: NoBTI) >
5780 std::distance(first: NoBTI, last: RepeatedSequenceLocs.end()))
5781 RepeatedSequenceLocs.erase(first: NoBTI, last: RepeatedSequenceLocs.end());
5782 else
5783 RepeatedSequenceLocs.erase(first: RepeatedSequenceLocs.begin(), last: NoBTI);
5784
5785 if (RepeatedSequenceLocs.size() < MinRepeats)
5786 return std::nullopt;
5787
5788 // Likewise, partition the candidates according to PAC-RET enablement.
5789 auto NoPAC =
5790 llvm::partition(Range&: RepeatedSequenceLocs, P: [](const outliner::Candidate &C) {
5791 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5792 // If the function happens to not spill the LR, do not disqualify it
5793 // from the outlining.
5794 return AFI.shouldSignReturnAddress(SpillsLR: true);
5795 });
5796 if (std::distance(first: RepeatedSequenceLocs.begin(), last: NoPAC) >
5797 std::distance(first: NoPAC, last: RepeatedSequenceLocs.end()))
5798 RepeatedSequenceLocs.erase(first: NoPAC, last: RepeatedSequenceLocs.end());
5799 else
5800 RepeatedSequenceLocs.erase(first: RepeatedSequenceLocs.begin(), last: NoPAC);
5801
5802 if (RepeatedSequenceLocs.size() < MinRepeats)
5803 return std::nullopt;
5804
5805 // At this point, we have only "safe" candidates to outline. Figure out
5806 // frame + call instruction information.
5807
5808 // Helper lambda which sets call information for every candidate.
5809 auto SetCandidateCallInfo =
5810 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5811 for (outliner::Candidate &C : RepeatedSequenceLocs)
5812 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
5813 };
5814
5815 OutlinerCosts Costs(Subtarget);
5816
5817 const auto &SomeMFI =
5818 *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>();
5819 // Adjust costs to account for the BTI instructions.
5820 if (SomeMFI.branchTargetEnforcement()) {
5821 Costs.FrameDefault += 4;
5822 Costs.FrameNoLRSave += 4;
5823 Costs.FrameRegSave += 4;
5824 Costs.FrameTailCall += 4;
5825 Costs.FrameThunk += 4;
5826 }
5827
5828 // Adjust costs to account for sign and authentication instructions.
5829 if (SomeMFI.shouldSignReturnAddress(SpillsLR: true)) {
5830 Costs.CallDefault += 8; // +PAC instr, +AUT instr
5831 Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr
5832 }
5833
5834 unsigned FrameID = MachineOutlinerDefault;
5835 unsigned NumBytesToCreateFrame = Costs.FrameDefault;
5836
5837 // If the last instruction in any candidate is a terminator, then we should
5838 // tail call all of the candidates.
5839 if (RepeatedSequenceLocs[0].back().isTerminator()) {
5840 FrameID = MachineOutlinerTailCall;
5841 NumBytesToCreateFrame = Costs.FrameTailCall;
5842 SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall);
5843 } else if (CanTransformInstrIntoTailCall(MI: RepeatedSequenceLocs[0].back())) {
5844 FrameID = MachineOutlinerThunk;
5845 NumBytesToCreateFrame = Costs.FrameThunk;
5846 SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk);
5847 } else {
5848 // We need to decide how to emit calls + frames. We can always emit the same
5849 // frame if we don't need to save to the stack. If we have to save to the
5850 // stack, then we need a different frame.
5851 unsigned NumBytesNoStackCalls = 0;
5852 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
5853
5854 for (outliner::Candidate &C : RepeatedSequenceLocs) {
5855 // LR liveness is overestimated in return blocks, unless they end with a
5856 // tail call.
5857 const auto Last = C.getMBB()->rbegin();
5858 const bool LRIsAvailable =
5859 C.getMBB()->isReturnBlock() && !Last->isCall()
5860 ? isLRAvailable(TRI, I: Last,
5861 E: (MachineBasicBlock::reverse_iterator)C.begin())
5862 : C.isAvailableAcrossAndOutOfSeq(Reg: ARM::LR, TRI);
5863 if (LRIsAvailable) {
5864 FrameID = MachineOutlinerNoLRSave;
5865 NumBytesNoStackCalls += Costs.CallNoLRSave;
5866 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: Costs.CallNoLRSave);
5867 CandidatesWithoutStackFixups.push_back(x: C);
5868 }
5869
5870 // Is an unused register available? If so, we won't modify the stack, so
5871 // we can outline with the same frame type as those that don't save LR.
5872 else if (findRegisterToSaveLRTo(C)) {
5873 FrameID = MachineOutlinerRegSave;
5874 NumBytesNoStackCalls += Costs.CallRegSave;
5875 C.setCallInfo(CID: MachineOutlinerRegSave, CO: Costs.CallRegSave);
5876 CandidatesWithoutStackFixups.push_back(x: C);
5877 }
5878
5879 // Is SP used in the sequence at all? If not, we don't have to modify
5880 // the stack, so we are guaranteed to get the same frame.
5881 else if (C.isAvailableInsideSeq(Reg: ARM::SP, TRI)) {
5882 NumBytesNoStackCalls += Costs.CallDefault;
5883 C.setCallInfo(CID: MachineOutlinerDefault, CO: Costs.CallDefault);
5884 CandidatesWithoutStackFixups.push_back(x: C);
5885 }
5886
5887 // If we outline this, we need to modify the stack. Pretend we don't
5888 // outline this by saving all of its bytes.
5889 else
5890 NumBytesNoStackCalls += SequenceSize;
5891 }
5892
5893 // If there are no places where we have to save LR, then note that we don't
5894 // have to update the stack. Otherwise, give every candidate the default
5895 // call type
5896 if (NumBytesNoStackCalls <=
5897 RepeatedSequenceLocs.size() * Costs.CallDefault) {
5898 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
5899 FrameID = MachineOutlinerNoLRSave;
5900 if (RepeatedSequenceLocs.size() < MinRepeats)
5901 return std::nullopt;
5902 } else
5903 SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault);
5904 }
5905
5906 // Does every candidate's MBB contain a call? If so, then we might have a
5907 // call in the range.
5908 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
5909 // check if the range contains a call. These require a save + restore of
5910 // the link register.
5911 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5912 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
5913 P: [](const MachineInstr &MI) { return MI.isCall(); }))
5914 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5915
5916 // Handle the last instruction separately. If it is tail call, then the
5917 // last instruction is a call, we don't want to save + restore in this
5918 // case. However, it could be possible that the last instruction is a
5919 // call without it being valid to tail call this sequence. We should
5920 // consider this as well.
5921 else if (FrameID != MachineOutlinerThunk &&
5922 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
5923 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
5924 }
5925
5926 return std::make_unique<outliner::OutlinedFunction>(
5927 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
5928}
5929
5930bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI,
5931 int64_t Fixup,
5932 bool Updt) const {
5933 int SPIdx = MI->findRegisterUseOperandIdx(Reg: ARM::SP, /*TRI=*/nullptr);
5934 unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask);
5935 if (SPIdx < 0)
5936 // No SP operand
5937 return true;
5938 else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2))
5939 // If SP is not the base register we can't do much
5940 return false;
5941
5942 // Stack might be involved but addressing mode doesn't handle any offset.
5943 // Rq: AddrModeT1_[1|2|4] don't operate on SP
5944 if (AddrMode == ARMII::AddrMode1 || // Arithmetic instructions
5945 AddrMode == ARMII::AddrMode4 || // Load/Store Multiple
5946 AddrMode == ARMII::AddrMode6 || // Neon Load/Store Multiple
5947 AddrMode == ARMII::AddrModeT2_so || // SP can't be used as based register
5948 AddrMode == ARMII::AddrModeT2_pc || // PCrel access
5949 AddrMode == ARMII::AddrMode2 || // Used by PRE and POST indexed LD/ST
5950 AddrMode == ARMII::AddrModeT2_i7 || // v8.1-M MVE
5951 AddrMode == ARMII::AddrModeT2_i7s2 || // v8.1-M MVE
5952 AddrMode == ARMII::AddrModeT2_i7s4 || // v8.1-M sys regs VLDR/VSTR
5953 AddrMode == ARMII::AddrModeNone ||
5954 AddrMode == ARMII::AddrModeT2_i8 || // Pre/Post inc instructions
5955 AddrMode == ARMII::AddrModeT2_i8neg) // Always negative imm
5956 return false;
5957
5958 unsigned NumOps = MI->getDesc().getNumOperands();
5959 unsigned ImmIdx = NumOps - 3;
5960
5961 const MachineOperand &Offset = MI->getOperand(i: ImmIdx);
5962 assert(Offset.isImm() && "Is not an immediate");
5963 int64_t OffVal = Offset.getImm();
5964
5965 if (OffVal < 0)
5966 // Don't override data if the are below SP.
5967 return false;
5968
5969 unsigned NumBits = 0;
5970 unsigned Scale = 1;
5971
5972 switch (AddrMode) {
5973 case ARMII::AddrMode3:
5974 if (ARM_AM::getAM3Op(AM3Opc: OffVal) == ARM_AM::sub)
5975 return false;
5976 OffVal = ARM_AM::getAM3Offset(AM3Opc: OffVal);
5977 NumBits = 8;
5978 break;
5979 case ARMII::AddrMode5:
5980 if (ARM_AM::getAM5Op(AM5Opc: OffVal) == ARM_AM::sub)
5981 return false;
5982 OffVal = ARM_AM::getAM5Offset(AM5Opc: OffVal);
5983 NumBits = 8;
5984 Scale = 4;
5985 break;
5986 case ARMII::AddrMode5FP16:
5987 if (ARM_AM::getAM5FP16Op(AM5Opc: OffVal) == ARM_AM::sub)
5988 return false;
5989 OffVal = ARM_AM::getAM5FP16Offset(AM5Opc: OffVal);
5990 NumBits = 8;
5991 Scale = 2;
5992 break;
5993 case ARMII::AddrModeT2_i8pos:
5994 NumBits = 8;
5995 break;
5996 case ARMII::AddrModeT2_i8s4:
5997 // FIXME: Values are already scaled in this addressing mode.
5998 assert((Fixup & 3) == 0 && "Can't encode this offset!");
5999 NumBits = 10;
6000 break;
6001 case ARMII::AddrModeT2_ldrex:
6002 NumBits = 8;
6003 Scale = 4;
6004 break;
6005 case ARMII::AddrModeT2_i12:
6006 case ARMII::AddrMode_i12:
6007 NumBits = 12;
6008 break;
6009 case ARMII::AddrModeT1_s: // SP-relative LD/ST
6010 NumBits = 8;
6011 Scale = 4;
6012 break;
6013 default:
6014 llvm_unreachable("Unsupported addressing mode!");
6015 }
6016 // Make sure the offset is encodable for instructions that scale the
6017 // immediate.
6018 assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 &&
6019 "Can't encode this offset!");
6020 OffVal += Fixup / Scale;
6021
6022 unsigned Mask = (1 << NumBits) - 1;
6023
6024 if (OffVal <= Mask) {
6025 if (Updt)
6026 MI->getOperand(i: ImmIdx).setImm(OffVal);
6027 return true;
6028 }
6029
6030 return false;
6031}
6032
6033void ARMBaseInstrInfo::mergeOutliningCandidateAttributes(
6034 Function &F, std::vector<outliner::Candidate> &Candidates) const {
6035 outliner::Candidate &C = Candidates.front();
6036 // branch-target-enforcement is guaranteed to be consistent between all
6037 // candidates, so we only need to look at one.
6038 const Function &CFn = C.getMF()->getFunction();
6039 if (CFn.hasFnAttribute(Kind: "branch-target-enforcement"))
6040 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "branch-target-enforcement"));
6041
6042 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
6043 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
6044
6045 ARMGenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
6046}
6047
6048bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
6049 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6050 const Function &F = MF.getFunction();
6051
6052 // Can F be deduplicated by the linker? If it can, don't outline from it.
6053 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6054 return false;
6055
6056 // Don't outline from functions with section markings; the program could
6057 // expect that all the code is in the named section.
6058 // FIXME: Allow outlining from multiple functions with the same section
6059 // marking.
6060 if (F.hasSection())
6061 return false;
6062
6063 // FIXME: Thumb1 outlining is not handled
6064 if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
6065 return false;
6066
6067 // It's safe to outline from MF.
6068 return true;
6069}
6070
6071bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6072 unsigned &Flags) const {
6073 // Check if LR is available through all of the MBB. If it's not, then set
6074 // a flag.
6075 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6076 "Suitable Machine Function for outlining must track liveness");
6077
6078 LiveRegUnits LRU(getRegisterInfo());
6079
6080 for (MachineInstr &MI : llvm::reverse(C&: MBB))
6081 LRU.accumulate(MI);
6082
6083 // Check if each of the unsafe registers are available...
6084 bool R12AvailableInBlock = LRU.available(Reg: ARM::R12);
6085 bool CPSRAvailableInBlock = LRU.available(Reg: ARM::CPSR);
6086
6087 // If all of these are dead (and not live out), we know we don't have to check
6088 // them later.
6089 if (R12AvailableInBlock && CPSRAvailableInBlock)
6090 Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6091
6092 // Now, add the live outs to the set.
6093 LRU.addLiveOuts(MBB);
6094
6095 // If any of these registers is available in the MBB, but also a live out of
6096 // the block, then we know outlining is unsafe.
6097 if (R12AvailableInBlock && !LRU.available(Reg: ARM::R12))
6098 return false;
6099 if (CPSRAvailableInBlock && !LRU.available(Reg: ARM::CPSR))
6100 return false;
6101
6102 // Check if there's a call inside this MachineBasicBlock. If there is, then
6103 // set a flag.
6104 if (any_of(Range&: MBB, P: [](MachineInstr &MI) { return MI.isCall(); }))
6105 Flags |= MachineOutlinerMBBFlags::HasCalls;
6106
6107 // LR liveness is overestimated in return blocks.
6108
6109 bool LRIsAvailable =
6110 MBB.isReturnBlock() && !MBB.back().isCall()
6111 ? isLRAvailable(TRI: getRegisterInfo(), I: MBB.rbegin(), E: MBB.rend())
6112 : LRU.available(Reg: ARM::LR);
6113 if (!LRIsAvailable)
6114 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6115
6116 return true;
6117}
6118
6119outliner::InstrType
6120ARMBaseInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
6121 MachineBasicBlock::iterator &MIT,
6122 unsigned Flags) const {
6123 MachineInstr &MI = *MIT;
6124 const TargetRegisterInfo *TRI = &getRegisterInfo();
6125
6126 // PIC instructions contain labels, outlining them would break offset
6127 // computing. unsigned Opc = MI.getOpcode();
6128 unsigned Opc = MI.getOpcode();
6129 if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
6130 Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
6131 Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
6132 Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
6133 Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
6134 Opc == ARM::t2MOV_ga_pcrel)
6135 return outliner::InstrType::Illegal;
6136
6137 // Be conservative with ARMv8.1 MVE instructions.
6138 if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
6139 Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
6140 Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
6141 Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
6142 Opc == ARM::t2LoopEndDec)
6143 return outliner::InstrType::Illegal;
6144
6145 const MCInstrDesc &MCID = MI.getDesc();
6146 uint64_t MIFlags = MCID.TSFlags;
6147 if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
6148 return outliner::InstrType::Illegal;
6149
6150 // Is this a terminator for a basic block?
6151 if (MI.isTerminator())
6152 // TargetInstrInfo::getOutliningType has already filtered out anything
6153 // that would break this, so we can allow it here.
6154 return outliner::InstrType::Legal;
6155
6156 // Don't outline if link register or program counter value are used.
6157 if (MI.readsRegister(Reg: ARM::LR, TRI) || MI.readsRegister(Reg: ARM::PC, TRI))
6158 return outliner::InstrType::Illegal;
6159
6160 if (MI.isCall()) {
6161 // Get the function associated with the call. Look at each operand and find
6162 // the one that represents the calle and get its name.
6163 const Function *Callee = nullptr;
6164 for (const MachineOperand &MOP : MI.operands()) {
6165 if (MOP.isGlobal()) {
6166 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
6167 break;
6168 }
6169 }
6170
6171 // Dont't outline calls to "mcount" like functions, in particular Linux
6172 // kernel function tracing relies on it.
6173 if (Callee &&
6174 (Callee->getName() == "\01__gnu_mcount_nc" ||
6175 Callee->getName() == "\01mcount" || Callee->getName() == "__mcount"))
6176 return outliner::InstrType::Illegal;
6177
6178 // If we don't know anything about the callee, assume it depends on the
6179 // stack layout of the caller. In that case, it's only legal to outline
6180 // as a tail-call. Explicitly list the call instructions we know about so
6181 // we don't get unexpected results with call pseudo-instructions.
6182 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6183 if (CanTransformInstrIntoTailCall(MI))
6184 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6185
6186 if (!Callee)
6187 return UnknownCallOutlineType;
6188
6189 // We have a function we have information about. Check if it's something we
6190 // can safely outline.
6191 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
6192
6193 // We don't know what's going on with the callee at all. Don't touch it.
6194 if (!CalleeMF)
6195 return UnknownCallOutlineType;
6196
6197 // Check if we know anything about the callee saves on the function. If we
6198 // don't, then don't touch it, since that implies that we haven't computed
6199 // anything about its stack frame yet.
6200 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6201 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6202 MFI.getNumObjects() > 0)
6203 return UnknownCallOutlineType;
6204
6205 // At this point, we can say that CalleeMF ought to not pass anything on the
6206 // stack. Therefore, we can outline it.
6207 return outliner::InstrType::Legal;
6208 }
6209
6210 // Since calls are handled, don't touch LR or PC
6211 if (MI.modifiesRegister(Reg: ARM::LR, TRI) || MI.modifiesRegister(Reg: ARM::PC, TRI))
6212 return outliner::InstrType::Illegal;
6213
6214 // Does this use the stack?
6215 if (MI.modifiesRegister(Reg: ARM::SP, TRI) || MI.readsRegister(Reg: ARM::SP, TRI)) {
6216 // True if there is no chance that any outlined candidate from this range
6217 // could require stack fixups. That is, both
6218 // * LR is available in the range (No save/restore around call)
6219 // * The range doesn't include calls (No save/restore in outlined frame)
6220 // are true.
6221 // These conditions also ensure correctness of the return address
6222 // authentication - we insert sign and authentication instructions only if
6223 // we save/restore LR on stack, but then this condition ensures that the
6224 // outlined range does not modify the SP, therefore the SP value used for
6225 // signing is the same as the one used for authentication.
6226 // FIXME: This is very restrictive; the flags check the whole block,
6227 // not just the bit we will try to outline.
6228 bool MightNeedStackFixUp =
6229 (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere |
6230 MachineOutlinerMBBFlags::HasCalls));
6231
6232 if (!MightNeedStackFixUp)
6233 return outliner::InstrType::Legal;
6234
6235 // Any modification of SP will break our code to save/restore LR.
6236 // FIXME: We could handle some instructions which add a constant offset to
6237 // SP, with a bit more work.
6238 if (MI.modifiesRegister(Reg: ARM::SP, TRI))
6239 return outliner::InstrType::Illegal;
6240
6241 // At this point, we have a stack instruction that we might need to fix up.
6242 // up. We'll handle it if it's a load or store.
6243 if (checkAndUpdateStackOffset(MI: &MI, Fixup: Subtarget.getStackAlignment().value(),
6244 Updt: false))
6245 return outliner::InstrType::Legal;
6246
6247 // We can't fix it up, so don't outline it.
6248 return outliner::InstrType::Illegal;
6249 }
6250
6251 // Be conservative with IT blocks.
6252 if (MI.readsRegister(Reg: ARM::ITSTATE, TRI) ||
6253 MI.modifiesRegister(Reg: ARM::ITSTATE, TRI))
6254 return outliner::InstrType::Illegal;
6255
6256 // Don't outline CFI instructions.
6257 if (MI.isCFIInstruction())
6258 return outliner::InstrType::Illegal;
6259
6260 return outliner::InstrType::Legal;
6261}
6262
6263void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6264 for (MachineInstr &MI : MBB) {
6265 checkAndUpdateStackOffset(MI: &MI, Fixup: Subtarget.getStackAlignment().value(), Updt: true);
6266 }
6267}
6268
6269void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
6270 MachineBasicBlock::iterator It, bool CFI,
6271 bool Auth) const {
6272 int Align = std::max(a: Subtarget.getStackAlignment().value(), b: uint64_t(8));
6273 unsigned MIFlags = CFI ? MachineInstr::FrameSetup : 0;
6274 assert(Align >= 8 && Align <= 256);
6275 if (Auth) {
6276 assert(Subtarget.isThumb2());
6277 // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
6278 // sequence.
6279 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2PAC)).setMIFlags(MIFlags);
6280 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2STRD_PRE), DestReg: ARM::SP)
6281 .addReg(RegNo: ARM::R12, Flags: RegState::Kill)
6282 .addReg(RegNo: ARM::LR, Flags: RegState::Kill)
6283 .addReg(RegNo: ARM::SP)
6284 .addImm(Val: -Align)
6285 .add(MOs: predOps(Pred: ARMCC::AL))
6286 .setMIFlags(MIFlags);
6287 } else {
6288 unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
6289 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg: ARM::SP)
6290 .addReg(RegNo: ARM::LR, Flags: RegState::Kill)
6291 .addReg(RegNo: ARM::SP)
6292 .addImm(Val: -Align)
6293 .add(MOs: predOps(Pred: ARMCC::AL))
6294 .setMIFlags(MIFlags);
6295 }
6296
6297 if (!CFI)
6298 return;
6299
6300 // Add a CFI, saying CFA is offset by Align bytes from SP.
6301 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
6302 CFIBuilder.buildDefCFAOffset(Offset: Align);
6303
6304 // Add a CFI saying that the LR that we want to find is now higher than
6305 // before.
6306 int LROffset = Auth ? Align - 4 : Align;
6307 CFIBuilder.buildOffset(Reg: ARM::LR, Offset: -LROffset);
6308 if (Auth) {
6309 // Add a CFI for the location of the return address PAC.
6310 CFIBuilder.buildOffset(Reg: ARM::RA_AUTH_CODE, Offset: -Align);
6311 }
6312}
6313
6314void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
6315 MachineBasicBlock::iterator It,
6316 bool CFI, bool Auth) const {
6317 int Align = Subtarget.getStackAlignment().value();
6318 unsigned MIFlags = CFI ? MachineInstr::FrameDestroy : 0;
6319 if (Auth) {
6320 assert(Subtarget.isThumb2());
6321 // Restore return address PAC and LR.
6322 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2LDRD_POST))
6323 .addReg(RegNo: ARM::R12, Flags: RegState::Define)
6324 .addReg(RegNo: ARM::LR, Flags: RegState::Define)
6325 .addReg(RegNo: ARM::SP, Flags: RegState::Define)
6326 .addReg(RegNo: ARM::SP)
6327 .addImm(Val: Align)
6328 .add(MOs: predOps(Pred: ARMCC::AL))
6329 .setMIFlags(MIFlags);
6330 // LR authentication is after the CFI instructions, below.
6331 } else {
6332 unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
6333 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg: ARM::LR)
6334 .addReg(RegNo: ARM::SP, Flags: RegState::Define)
6335 .addReg(RegNo: ARM::SP);
6336 if (!Subtarget.isThumb())
6337 MIB.addReg(RegNo: 0);
6338 MIB.addImm(Val: Subtarget.getStackAlignment().value())
6339 .add(MOs: predOps(Pred: ARMCC::AL))
6340 .setMIFlags(MIFlags);
6341 }
6342
6343 if (CFI) {
6344 // Now stack has moved back up and we have restored LR.
6345 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameDestroy);
6346 CFIBuilder.buildDefCFAOffset(Offset: 0);
6347 CFIBuilder.buildRestore(Reg: ARM::LR);
6348 if (Auth)
6349 CFIBuilder.buildUndefined(Reg: ARM::RA_AUTH_CODE);
6350 }
6351
6352 if (Auth)
6353 BuildMI(BB&: MBB, I: It, MIMD: DebugLoc(), MCID: get(Opcode: ARM::t2AUT));
6354}
6355
6356void ARMBaseInstrInfo::buildOutlinedFrame(
6357 MachineBasicBlock &MBB, MachineFunction &MF,
6358 const outliner::OutlinedFunction &OF) const {
6359 // For thunk outlining, rewrite the last instruction from a call to a
6360 // tail-call.
6361 if (OF.FrameConstructionID == MachineOutlinerThunk) {
6362 MachineInstr *Call = &*--MBB.instr_end();
6363 bool isThumb = Subtarget.isThumb();
6364 unsigned FuncOp = isThumb ? 2 : 0;
6365 unsigned Opc = Call->getOperand(i: FuncOp).isReg()
6366 ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
6367 : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
6368 : ARM::tTAILJMPdND
6369 : ARM::TAILJMPd;
6370 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6371 .add(MO: Call->getOperand(i: FuncOp));
6372 if (isThumb && !Call->getOperand(i: FuncOp).isReg())
6373 MIB.add(MOs: predOps(Pred: ARMCC::AL));
6374 Call->eraseFromParent();
6375 }
6376
6377 // Is there a call in the outlined range?
6378 auto IsNonTailCall = [](MachineInstr &MI) {
6379 return MI.isCall() && !MI.isReturn();
6380 };
6381 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
6382 MachineBasicBlock::iterator It = MBB.begin();
6383 MachineBasicBlock::iterator Et = MBB.end();
6384
6385 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6386 OF.FrameConstructionID == MachineOutlinerThunk)
6387 Et = std::prev(x: MBB.end());
6388
6389 // We have to save and restore LR, we need to add it to the liveins if it
6390 // is not already part of the set. This is sufficient since outlined
6391 // functions only have one block.
6392 if (!MBB.isLiveIn(Reg: ARM::LR))
6393 MBB.addLiveIn(PhysReg: ARM::LR);
6394
6395 // Insert a save before the outlined region
6396 bool Auth = MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(SpillsLR: true);
6397 saveLROnStack(MBB, It, CFI: true, Auth);
6398
6399 // Fix up the instructions in the range, since we're going to modify the
6400 // stack.
6401 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6402 "Can only fix up stack references once");
6403 fixupPostOutline(MBB);
6404
6405 // Insert a restore before the terminator for the function. Restore LR.
6406 restoreLRFromStack(MBB, It: Et, CFI: true, Auth);
6407 }
6408
6409 // If this is a tail call outlined function, then there's already a return.
6410 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6411 OF.FrameConstructionID == MachineOutlinerThunk)
6412 return;
6413
6414 // Here we have to insert the return ourselves. Get the correct opcode from
6415 // current feature set.
6416 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: Subtarget.getReturnOpcode()))
6417 .add(MOs: predOps(Pred: ARMCC::AL));
6418
6419 // Did we have to modify the stack by saving the link register?
6420 if (OF.FrameConstructionID != MachineOutlinerDefault &&
6421 OF.Candidates[0].CallConstructionID != MachineOutlinerDefault)
6422 return;
6423
6424 // We modified the stack.
6425 // Walk over the basic block and fix up all the stack accesses.
6426 fixupPostOutline(MBB);
6427}
6428
6429MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
6430 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6431 MachineFunction &MF, outliner::Candidate &C) const {
6432 MachineInstrBuilder MIB;
6433 MachineBasicBlock::iterator CallPt;
6434 unsigned Opc;
6435 bool isThumb = Subtarget.isThumb();
6436
6437 // Are we tail calling?
6438 if (C.CallConstructionID == MachineOutlinerTailCall) {
6439 // If yes, then we can just branch to the label.
6440 Opc = isThumb
6441 ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
6442 : ARM::TAILJMPd;
6443 MIB = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6444 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()));
6445 if (isThumb)
6446 MIB.add(MOs: predOps(Pred: ARMCC::AL));
6447 It = MBB.insert(I: It, MI: MIB);
6448 return It;
6449 }
6450
6451 // Create the call instruction.
6452 Opc = isThumb ? ARM::tBL : ARM::BL;
6453 MachineInstrBuilder CallMIB = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: Opc));
6454 if (isThumb)
6455 CallMIB.add(MOs: predOps(Pred: ARMCC::AL));
6456 CallMIB.addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()));
6457
6458 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6459 C.CallConstructionID == MachineOutlinerThunk) {
6460 // No, so just insert the call.
6461 It = MBB.insert(I: It, MI: CallMIB);
6462 return It;
6463 }
6464
6465 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
6466 // Can we save to a register?
6467 if (C.CallConstructionID == MachineOutlinerRegSave) {
6468 Register Reg = findRegisterToSaveLRTo(C);
6469 assert(Reg != 0 && "No callee-saved register available?");
6470
6471 // Save and restore LR from that register.
6472 copyPhysReg(MBB, I: It, DL: DebugLoc(), DestReg: Reg, SrcReg: ARM::LR, KillSrc: true);
6473 if (!AFI.isLRSpilled())
6474 CFIInstBuilder(MBB, It, MachineInstr::FrameSetup)
6475 .buildRegister(Reg1: ARM::LR, Reg2: Reg);
6476 CallPt = MBB.insert(I: It, MI: CallMIB);
6477 copyPhysReg(MBB, I: It, DL: DebugLoc(), DestReg: ARM::LR, SrcReg: Reg, KillSrc: true);
6478 if (!AFI.isLRSpilled())
6479 CFIInstBuilder(MBB, It, MachineInstr::FrameDestroy).buildRestore(Reg: ARM::LR);
6480 It--;
6481 return CallPt;
6482 }
6483 // We have the default case. Save and restore from SP.
6484 if (!MBB.isLiveIn(Reg: ARM::LR))
6485 MBB.addLiveIn(PhysReg: ARM::LR);
6486 bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(SpillsLR: true);
6487 saveLROnStack(MBB, It, CFI: !AFI.isLRSpilled(), Auth);
6488 CallPt = MBB.insert(I: It, MI: CallMIB);
6489 restoreLRFromStack(MBB, It, CFI: !AFI.isLRSpilled(), Auth);
6490 It--;
6491 return CallPt;
6492}
6493
6494bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault(
6495 MachineFunction &MF) const {
6496 return Subtarget.isMClass() && MF.getFunction().hasMinSize();
6497}
6498
6499bool ARMBaseInstrInfo::isReMaterializableImpl(
6500 const MachineInstr &MI) const {
6501 // Try hard to rematerialize any VCTPs because if we spill P0, it will block
6502 // the tail predication conversion. This means that the element count
6503 // register has to be live for longer, but that has to be better than
6504 // spill/restore and VPT predication.
6505 return (isVCTP(MI: &MI) && !isPredicated(MI)) ||
6506 TargetInstrInfo::isReMaterializableImpl(MI);
6507}
6508
6509unsigned llvm::getBLXOpcode(const MachineFunction &MF) {
6510 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip
6511 : ARM::BLX;
6512}
6513
6514unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) {
6515 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip
6516 : ARM::tBLXr;
6517}
6518
6519unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) {
6520 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip
6521 : ARM::BLX_pred;
6522}
6523
6524namespace {
6525class ARMPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
6526 MachineInstr *EndLoop, *LoopCount;
6527 MachineFunction *MF;
6528 const TargetInstrInfo *TII;
6529
6530 // Bitset[0 .. MAX_STAGES-1] ... iterations needed
6531 // [LAST_IS_USE] : last reference to register in schedule is a use
6532 // [SEEN_AS_LIVE] : Normal pressure algorithm believes register is live
6533 static int constexpr MAX_STAGES = 30;
6534 static int constexpr LAST_IS_USE = MAX_STAGES;
6535 static int constexpr SEEN_AS_LIVE = MAX_STAGES + 1;
6536 typedef std::bitset<MAX_STAGES + 2> IterNeed;
6537 typedef std::map<Register, IterNeed> IterNeeds;
6538
6539 void bumpCrossIterationPressure(RegPressureTracker &RPT,
6540 const IterNeeds &CIN);
6541 bool tooMuchRegisterPressure(SwingSchedulerDAG &SSD, SMSchedule &SMS);
6542
6543 // Meanings of the various stuff with loop types:
6544 // t2Bcc:
6545 // EndLoop = branch at end of original BB that will become a kernel
6546 // LoopCount = CC setter live into branch
6547 // t2LoopEnd:
6548 // EndLoop = branch at end of original BB
6549 // LoopCount = t2LoopDec
6550public:
6551 ARMPipelinerLoopInfo(MachineInstr *EndLoop, MachineInstr *LoopCount)
6552 : EndLoop(EndLoop), LoopCount(LoopCount),
6553 MF(EndLoop->getParent()->getParent()),
6554 TII(MF->getSubtarget().getInstrInfo()) {}
6555
6556 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
6557 // Only ignore the terminator.
6558 return MI == EndLoop || MI == LoopCount;
6559 }
6560
6561 bool shouldUseSchedule(SwingSchedulerDAG &SSD, SMSchedule &SMS) override {
6562 if (tooMuchRegisterPressure(SSD, SMS))
6563 return false;
6564
6565 return true;
6566 }
6567
6568 std::optional<bool> createTripCountGreaterCondition(
6569 int TC, MachineBasicBlock &MBB,
6570 SmallVectorImpl<MachineOperand> &Cond) override {
6571
6572 if (isCondBranchOpcode(Opc: EndLoop->getOpcode())) {
6573 Cond.push_back(Elt: EndLoop->getOperand(i: 1));
6574 Cond.push_back(Elt: EndLoop->getOperand(i: 2));
6575 if (EndLoop->getOperand(i: 0).getMBB() == EndLoop->getParent()) {
6576 TII->reverseBranchCondition(Cond);
6577 }
6578 return {};
6579 } else if (EndLoop->getOpcode() == ARM::t2LoopEnd) {
6580 // General case just lets the unrolled t2LoopDec do the subtraction and
6581 // therefore just needs to check if zero has been reached.
6582 MachineInstr *LoopDec = nullptr;
6583 for (auto &I : MBB.instrs())
6584 if (I.getOpcode() == ARM::t2LoopDec)
6585 LoopDec = &I;
6586 assert(LoopDec && "Unable to find copied LoopDec");
6587 // Check if we're done with the loop.
6588 BuildMI(BB: &MBB, MIMD: LoopDec->getDebugLoc(), MCID: TII->get(Opcode: ARM::t2CMPri))
6589 .addReg(RegNo: LoopDec->getOperand(i: 0).getReg())
6590 .addImm(Val: 0)
6591 .addImm(Val: ARMCC::AL)
6592 .addReg(RegNo: ARM::NoRegister);
6593 Cond.push_back(Elt: MachineOperand::CreateImm(Val: ARMCC::EQ));
6594 Cond.push_back(Elt: MachineOperand::CreateReg(Reg: ARM::CPSR, isDef: false));
6595 return {};
6596 } else
6597 llvm_unreachable("Unknown EndLoop");
6598 }
6599
6600 void setPreheader(MachineBasicBlock *NewPreheader) override {}
6601
6602 void adjustTripCount(int TripCountAdjust) override {}
6603};
6604
6605void ARMPipelinerLoopInfo::bumpCrossIterationPressure(RegPressureTracker &RPT,
6606 const IterNeeds &CIN) {
6607 // Increase pressure by the amounts in CrossIterationNeeds
6608 for (const auto &N : CIN) {
6609 int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6610 for (int I = 0; I < Cnt; ++I)
6611 RPT.increaseRegPressure(VRegOrUnit: VirtRegOrUnit(N.first), PreviousMask: LaneBitmask::getNone(),
6612 NewMask: LaneBitmask::getAll());
6613 }
6614 // Decrease pressure by the amounts in CrossIterationNeeds
6615 for (const auto &N : CIN) {
6616 int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6617 for (int I = 0; I < Cnt; ++I)
6618 RPT.decreaseRegPressure(VRegOrUnit: VirtRegOrUnit(N.first), PreviousMask: LaneBitmask::getAll(),
6619 NewMask: LaneBitmask::getNone());
6620 }
6621}
6622
6623bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
6624 SMSchedule &SMS) {
6625 IterNeeds CrossIterationNeeds;
6626
6627 // Determine which values will be loop-carried after the schedule is
6628 // applied
6629
6630 for (auto &SU : SSD.SUnits) {
6631 const MachineInstr *MI = SU.getInstr();
6632 int Stg = SMS.stageScheduled(SU: const_cast<SUnit *>(&SU));
6633 for (auto &S : SU.Succs)
6634 if (MI->isPHI() && S.getKind() == SDep::Anti) {
6635 Register Reg = S.getReg();
6636 if (Reg.isVirtual())
6637 CrossIterationNeeds[Reg.id()].set(position: 0);
6638 } else if (S.isAssignedRegDep()) {
6639 int OStg = SMS.stageScheduled(SU: S.getSUnit());
6640 if (OStg >= 0 && OStg != Stg) {
6641 Register Reg = S.getReg();
6642 if (Reg.isVirtual())
6643 CrossIterationNeeds[Reg.id()] |= ((1 << (OStg - Stg)) - 1);
6644 }
6645 }
6646 }
6647
6648 // Determine more-or-less what the proposed schedule (reversed) is going to
6649 // be; it might not be quite the same because the within-cycle ordering
6650 // created by SMSchedule depends upon changes to help with address offsets and
6651 // the like.
6652 std::vector<SUnit *> ProposedSchedule;
6653 for (int Cycle = SMS.getFinalCycle(); Cycle >= SMS.getFirstCycle(); --Cycle)
6654 for (int Stage = 0, StageEnd = SMS.getMaxStageCount(); Stage <= StageEnd;
6655 ++Stage) {
6656 std::deque<SUnit *> Instrs =
6657 SMS.getInstructions(cycle: Cycle + Stage * SMS.getInitiationInterval());
6658 std::sort(first: Instrs.begin(), last: Instrs.end(),
6659 comp: [](SUnit *A, SUnit *B) { return A->NodeNum > B->NodeNum; });
6660 llvm::append_range(C&: ProposedSchedule, R&: Instrs);
6661 }
6662
6663 // Learn whether the last use/def of each cross-iteration register is a use or
6664 // def. If it is a def, RegisterPressure will implicitly increase max pressure
6665 // and we do not have to add the pressure.
6666 for (auto *SU : ProposedSchedule)
6667 for (ConstMIBundleOperands OperI(*SU->getInstr()); OperI.isValid();
6668 ++OperI) {
6669 auto MO = *OperI;
6670 if (!MO.isReg() || !MO.getReg())
6671 continue;
6672 Register Reg = MO.getReg();
6673 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6674 if (CIter == CrossIterationNeeds.end() || CIter->second[LAST_IS_USE] ||
6675 CIter->second[SEEN_AS_LIVE])
6676 continue;
6677 if (MO.isDef() && !MO.isDead())
6678 CIter->second.set(position: SEEN_AS_LIVE);
6679 else if (MO.isUse())
6680 CIter->second.set(position: LAST_IS_USE);
6681 }
6682 for (auto &CI : CrossIterationNeeds)
6683 CI.second.reset(position: LAST_IS_USE);
6684
6685 RegionPressure RecRegPressure;
6686 RegPressureTracker RPTracker(RecRegPressure);
6687 RegisterClassInfo RegClassInfo;
6688 RegClassInfo.runOnMachineFunction(MF: *MF);
6689 RPTracker.init(mf: MF, rci: &RegClassInfo, lis: nullptr, mbb: EndLoop->getParent(),
6690 pos: EndLoop->getParent()->end(), TrackLaneMasks: false, TrackUntiedDefs: false);
6691
6692 bumpCrossIterationPressure(RPT&: RPTracker, CIN: CrossIterationNeeds);
6693
6694 for (auto *SU : ProposedSchedule) {
6695 MachineBasicBlock::const_iterator CurInstI = SU->getInstr();
6696 RPTracker.setPos(std::next(x: CurInstI));
6697 RPTracker.recede();
6698
6699 // Track what cross-iteration registers would be seen as live
6700 for (ConstMIBundleOperands OperI(*CurInstI); OperI.isValid(); ++OperI) {
6701 auto MO = *OperI;
6702 if (!MO.isReg() || !MO.getReg())
6703 continue;
6704 Register Reg = MO.getReg();
6705 if (MO.isDef() && !MO.isDead()) {
6706 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6707 if (CIter != CrossIterationNeeds.end()) {
6708 CIter->second.reset(position: 0);
6709 CIter->second.reset(position: SEEN_AS_LIVE);
6710 }
6711 }
6712 }
6713 for (auto &S : SU->Preds) {
6714 auto Stg = SMS.stageScheduled(SU);
6715 if (S.isAssignedRegDep()) {
6716 Register Reg = S.getReg();
6717 auto CIter = CrossIterationNeeds.find(x: Reg.id());
6718 if (CIter != CrossIterationNeeds.end()) {
6719 auto Stg2 = SMS.stageScheduled(SU: S.getSUnit());
6720 assert(Stg2 <= Stg && "Data dependence upon earlier stage");
6721 if (Stg - Stg2 < MAX_STAGES)
6722 CIter->second.set(position: Stg - Stg2);
6723 CIter->second.set(position: SEEN_AS_LIVE);
6724 }
6725 }
6726 }
6727
6728 bumpCrossIterationPressure(RPT&: RPTracker, CIN: CrossIterationNeeds);
6729 }
6730
6731 auto &P = RPTracker.getPressure().MaxSetPressure;
6732 for (unsigned I = 0, E = P.size(); I < E; ++I) {
6733 // Exclude some Neon register classes.
6734 if (I == ARM::DQuad_with_ssub_0 || I == ARM::DTripleSpc_with_ssub_0 ||
6735 I == ARM::DTriple_with_qsub_0_in_QPR)
6736 continue;
6737
6738 if (P[I] > RegClassInfo.getRegPressureSetLimit(Idx: I)) {
6739 return true;
6740 }
6741 }
6742 return false;
6743}
6744
6745} // namespace
6746
6747std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
6748ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
6749 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
6750 MachineBasicBlock *Preheader = *LoopBB->pred_begin();
6751 if (Preheader == LoopBB)
6752 Preheader = *std::next(x: LoopBB->pred_begin());
6753
6754 if (I != LoopBB->end() && I->getOpcode() == ARM::t2Bcc) {
6755 // If the branch is a Bcc, then the CPSR should be set somewhere within the
6756 // block. We need to determine the reaching definition of CPSR so that
6757 // it can be marked as non-pipelineable, allowing the pipeliner to force
6758 // it into stage 0 or give up if it cannot or will not do so.
6759 MachineInstr *CCSetter = nullptr;
6760 for (auto &L : LoopBB->instrs()) {
6761 if (L.isCall())
6762 return nullptr;
6763 if (isCPSRDefined(MI: L))
6764 CCSetter = &L;
6765 }
6766 if (CCSetter)
6767 return std::make_unique<ARMPipelinerLoopInfo>(args: &*I, args&: CCSetter);
6768 else
6769 return nullptr; // Unable to find the CC setter, so unable to guarantee
6770 // that pipeline will work
6771 }
6772
6773 // Recognize:
6774 // preheader:
6775 // %1 = t2DoopLoopStart %0
6776 // loop:
6777 // %2 = phi %1, <not loop>, %..., %loop
6778 // %3 = t2LoopDec %2, <imm>
6779 // t2LoopEnd %3, %loop
6780
6781 if (I != LoopBB->end() && I->getOpcode() == ARM::t2LoopEnd) {
6782 for (auto &L : LoopBB->instrs())
6783 if (L.isCall())
6784 return nullptr;
6785 else if (isVCTP(MI: &L))
6786 return nullptr;
6787 Register LoopDecResult = I->getOperand(i: 0).getReg();
6788 MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
6789 MachineInstr *LoopDec = MRI.getUniqueVRegDef(Reg: LoopDecResult);
6790 if (!LoopDec || LoopDec->getOpcode() != ARM::t2LoopDec)
6791 return nullptr;
6792 MachineInstr *LoopStart = nullptr;
6793 for (auto &J : Preheader->instrs())
6794 if (J.getOpcode() == ARM::t2DoLoopStart)
6795 LoopStart = &J;
6796 if (!LoopStart)
6797 return nullptr;
6798 return std::make_unique<ARMPipelinerLoopInfo>(args: &*I, args&: LoopDec);
6799 }
6800 return nullptr;
6801}
6802