1//=== A15SDOptimizerPass.cpp - Optimize DPR and SPR register accesses on A15==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The Cortex-A15 processor employs a tracking scheme in its register renaming
10// in order to process each instruction's micro-ops speculatively and
11// out-of-order with appropriate forwarding. The ARM architecture allows VFP
12// instructions to read and write 32-bit S-registers. Each S-register
13// corresponds to one half (upper or lower) of an overlaid 64-bit D-register.
14//
15// There are several instruction patterns which can be used to provide this
16// capability which can provide higher performance than other, potentially more
17// direct patterns, specifically around when one micro-op reads a D-register
18// operand that has recently been written as one or more S-register results.
19//
20// This file defines a pre-regalloc pass which looks for SPR producers which
21// are going to be used by a DPR (or QPR) consumers and creates the more
22// optimized access pattern.
23//
24//===----------------------------------------------------------------------===//
25
26#include "ARM.h"
27#include "ARMBaseInstrInfo.h"
28#include "ARMBaseRegisterInfo.h"
29#include "ARMSubtarget.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineRegisterInfo.h"
36#include "llvm/CodeGen/RegisterClassInfo.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/CodeGen/TargetSubtargetInfo.h"
39#include "llvm/Support/Debug.h"
40#include "llvm/Support/raw_ostream.h"
41#include <map>
42#include <set>
43
44using namespace llvm;
45
46#define DEBUG_TYPE "a15-sd-optimizer"
47
48namespace {
49 struct A15SDOptimizer : public MachineFunctionPass {
50 static char ID;
51 A15SDOptimizer() : MachineFunctionPass(ID) {}
52
53 bool runOnMachineFunction(MachineFunction &Fn) override;
54
55 StringRef getPassName() const override { return "ARM A15 S->D optimizer"; }
56
57 void getAnalysisUsage(AnalysisUsage &AU) const override {
58 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
59 MachineFunctionPass::getAnalysisUsage(AU);
60 }
61
62 private:
63 const ARMBaseInstrInfo *TII;
64 const TargetRegisterInfo *TRI;
65 MachineRegisterInfo *MRI;
66
67 bool runOnInstruction(MachineInstr *MI);
68
69 //
70 // Instruction builder helpers
71 //
72 unsigned createDupLane(MachineBasicBlock &MBB,
73 MachineBasicBlock::iterator InsertBefore,
74 const DebugLoc &DL, unsigned Reg, unsigned Lane,
75 bool QPR = false);
76
77 unsigned createExtractSubreg(MachineBasicBlock &MBB,
78 MachineBasicBlock::iterator InsertBefore,
79 const DebugLoc &DL, unsigned DReg,
80 unsigned Lane, const TargetRegisterClass *TRC);
81
82 unsigned createVExt(MachineBasicBlock &MBB,
83 MachineBasicBlock::iterator InsertBefore,
84 const DebugLoc &DL, unsigned Ssub0, unsigned Ssub1);
85
86 unsigned createRegSequence(MachineBasicBlock &MBB,
87 MachineBasicBlock::iterator InsertBefore,
88 const DebugLoc &DL, unsigned Reg1,
89 unsigned Reg2);
90
91 unsigned createInsertSubreg(MachineBasicBlock &MBB,
92 MachineBasicBlock::iterator InsertBefore,
93 const DebugLoc &DL, unsigned DReg,
94 unsigned Lane, unsigned ToInsert);
95
96 unsigned createImplicitDef(MachineBasicBlock &MBB,
97 MachineBasicBlock::iterator InsertBefore,
98 const DebugLoc &DL);
99
100 //
101 // Various property checkers
102 //
103 bool usesRegClass(MachineOperand &MO, const TargetRegisterClass *TRC);
104 bool hasPartialWrite(MachineInstr *MI);
105 SmallVector<unsigned, 8> getReadDPRs(MachineInstr *MI);
106 unsigned getDPRLaneFromSPR(unsigned SReg);
107
108 //
109 // Methods used for getting the definitions of partial registers
110 //
111
112 MachineInstr *elideCopies(MachineInstr *MI);
113 void elideCopiesAndPHIs(MachineInstr *MI,
114 SmallVectorImpl<MachineInstr*> &Outs);
115
116 //
117 // Pattern optimization methods
118 //
119 unsigned optimizeAllLanesPattern(MachineInstr *MI, unsigned Reg);
120 unsigned optimizeSDPattern(MachineInstr *MI);
121 unsigned getPrefSPRLane(unsigned SReg);
122
123 //
124 // Sanitizing method - used to make sure if don't leave dead code around.
125 //
126 void eraseInstrWithNoUses(MachineInstr *MI);
127
128 //
129 // A map used to track the changes done by this pass.
130 //
131 std::map<MachineInstr*, unsigned> Replacements;
132 std::set<MachineInstr *> DeadInstr;
133 };
134 char A15SDOptimizer::ID = 0;
135} // end anonymous namespace
136
137// Returns true if this is a use of a SPR register.
138bool A15SDOptimizer::usesRegClass(MachineOperand &MO,
139 const TargetRegisterClass *TRC) {
140 if (!MO.isReg())
141 return false;
142 Register Reg = MO.getReg();
143
144 if (Reg.isVirtual())
145 return MRI->getRegClass(Reg)->hasSuperClassEq(RC: TRC);
146 else
147 return TRC->contains(Reg);
148}
149
150unsigned A15SDOptimizer::getDPRLaneFromSPR(unsigned SReg) {
151 MCRegister DReg =
152 TRI->getMatchingSuperReg(Reg: SReg, SubIdx: ARM::ssub_1, RC: &ARM::DPRRegClass);
153 if (DReg)
154 return ARM::ssub_1;
155 return ARM::ssub_0;
156}
157
158// Get the subreg type that is most likely to be coalesced
159// for an SPR register that will be used in VDUP32d pseudo.
160unsigned A15SDOptimizer::getPrefSPRLane(unsigned SReg) {
161 if (!Register::isVirtualRegister(Reg: SReg))
162 return getDPRLaneFromSPR(SReg);
163
164 MachineInstr *MI = MRI->getVRegDef(Reg: SReg);
165 if (!MI) return ARM::ssub_0;
166 MachineOperand *MO = MI->findRegisterDefOperand(Reg: SReg, /*TRI=*/nullptr);
167 if (!MO) return ARM::ssub_0;
168 assert(MO->isReg() && "Non-register operand found!");
169
170 if (MI->isCopy() && usesRegClass(MO&: MI->getOperand(i: 1),
171 TRC: &ARM::SPRRegClass)) {
172 SReg = MI->getOperand(i: 1).getReg();
173 }
174
175 if (Register::isVirtualRegister(Reg: SReg)) {
176 if (MO->getSubReg() == ARM::ssub_1) return ARM::ssub_1;
177 return ARM::ssub_0;
178 }
179 return getDPRLaneFromSPR(SReg);
180}
181
182// MI is known to be dead. Figure out what instructions
183// are also made dead by this and mark them for removal.
184void A15SDOptimizer::eraseInstrWithNoUses(MachineInstr *MI) {
185 SmallVector<MachineInstr *, 8> Front;
186 DeadInstr.insert(x: MI);
187
188 LLVM_DEBUG(dbgs() << "Deleting base instruction " << *MI << "\n");
189 Front.push_back(Elt: MI);
190
191 while (Front.size() != 0) {
192 MI = Front.pop_back_val();
193
194 // MI is already known to be dead. We need to see
195 // if other instructions can also be removed.
196 for (MachineOperand &MO : MI->operands()) {
197 if ((!MO.isReg()) || (!MO.isUse()))
198 continue;
199 Register Reg = MO.getReg();
200 if (!Reg.isVirtual())
201 continue;
202 MachineOperand *Op = MI->findRegisterDefOperand(Reg, /*TRI=*/nullptr);
203
204 if (!Op)
205 continue;
206
207 MachineInstr *Def = Op->getParent();
208
209 // We don't need to do anything if we have already marked
210 // this instruction as being dead.
211 if (DeadInstr.find(x: Def) != DeadInstr.end())
212 continue;
213
214 // Check if all the uses of this instruction are marked as
215 // dead. If so, we can also mark this instruction as being
216 // dead.
217 bool IsDead = true;
218 for (MachineOperand &MODef : Def->operands()) {
219 if ((!MODef.isReg()) || (!MODef.isDef()))
220 continue;
221 Register DefReg = MODef.getReg();
222 if (!DefReg.isVirtual()) {
223 IsDead = false;
224 break;
225 }
226 for (MachineInstr &Use : MRI->use_instructions(Reg)) {
227 // We don't care about self references.
228 if (&Use == Def)
229 continue;
230 if (DeadInstr.find(x: &Use) == DeadInstr.end()) {
231 IsDead = false;
232 break;
233 }
234 }
235 }
236
237 if (!IsDead) continue;
238
239 LLVM_DEBUG(dbgs() << "Deleting instruction " << *Def << "\n");
240 DeadInstr.insert(x: Def);
241 }
242 }
243}
244
245// Creates the more optimized patterns and generally does all the code
246// transformations in this pass.
247unsigned A15SDOptimizer::optimizeSDPattern(MachineInstr *MI) {
248 if (MI->isCopy()) {
249 return optimizeAllLanesPattern(MI, Reg: MI->getOperand(i: 1).getReg());
250 }
251
252 if (MI->isInsertSubreg()) {
253 Register DPRReg = MI->getOperand(i: 1).getReg();
254 Register SPRReg = MI->getOperand(i: 2).getReg();
255
256 if (DPRReg.isVirtual() && SPRReg.isVirtual()) {
257 MachineInstr *DPRMI = MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg());
258 MachineInstr *SPRMI = MRI->getVRegDef(Reg: MI->getOperand(i: 2).getReg());
259
260 if (DPRMI && SPRMI) {
261 // See if the first operand of this insert_subreg is IMPLICIT_DEF
262 MachineInstr *ECDef = elideCopies(MI: DPRMI);
263 if (ECDef && ECDef->isImplicitDef()) {
264 // Another corner case - if we're inserting something that is purely
265 // a subreg copy of a DPR, just use that DPR.
266
267 MachineInstr *EC = elideCopies(MI: SPRMI);
268 // Is it a subreg copy of ssub_0?
269 if (EC && EC->isCopy() &&
270 EC->getOperand(i: 1).getSubReg() == ARM::ssub_0) {
271 LLVM_DEBUG(dbgs() << "Found a subreg copy: " << *SPRMI);
272
273 // Find the thing we're subreg copying out of - is it of the same
274 // regclass as DPRMI? (i.e. a DPR or QPR).
275 Register FullReg = SPRMI->getOperand(i: 1).getReg();
276 const TargetRegisterClass *TRC =
277 MRI->getRegClass(Reg: MI->getOperand(i: 1).getReg());
278 if (TRC->hasSuperClassEq(RC: MRI->getRegClass(Reg: FullReg))) {
279 LLVM_DEBUG(dbgs() << "Subreg copy is compatible - returning ");
280 LLVM_DEBUG(dbgs() << printReg(FullReg) << "\n");
281 eraseInstrWithNoUses(MI);
282 return FullReg;
283 }
284 }
285
286 return optimizeAllLanesPattern(MI, Reg: MI->getOperand(i: 2).getReg());
287 }
288 }
289 }
290 return optimizeAllLanesPattern(MI, Reg: MI->getOperand(i: 0).getReg());
291 }
292
293 if (MI->isRegSequence() && usesRegClass(MO&: MI->getOperand(i: 1),
294 TRC: &ARM::SPRRegClass)) {
295 // See if all bar one of the operands are IMPLICIT_DEF and insert the
296 // optimizer pattern accordingly.
297 unsigned NumImplicit = 0, NumTotal = 0;
298 unsigned NonImplicitReg = ~0U;
299
300 for (MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->explicit_operands())) {
301 if (!MO.isReg())
302 continue;
303 ++NumTotal;
304 Register OpReg = MO.getReg();
305
306 if (!OpReg.isVirtual())
307 break;
308
309 MachineInstr *Def = MRI->getVRegDef(Reg: OpReg);
310 if (!Def)
311 break;
312 if (Def->isImplicitDef())
313 ++NumImplicit;
314 else
315 NonImplicitReg = MO.getReg();
316 }
317
318 if (NumImplicit == NumTotal - 1)
319 return optimizeAllLanesPattern(MI, Reg: NonImplicitReg);
320 else
321 return optimizeAllLanesPattern(MI, Reg: MI->getOperand(i: 0).getReg());
322 }
323
324 llvm_unreachable("Unhandled update pattern!");
325}
326
327// Return true if this MachineInstr inserts a scalar (SPR) value into
328// a D or Q register.
329bool A15SDOptimizer::hasPartialWrite(MachineInstr *MI) {
330 // The only way we can do a partial register update is through a COPY,
331 // INSERT_SUBREG or REG_SEQUENCE.
332 if (MI->isCopy() && usesRegClass(MO&: MI->getOperand(i: 1), TRC: &ARM::SPRRegClass))
333 return true;
334
335 if (MI->isInsertSubreg() && usesRegClass(MO&: MI->getOperand(i: 2),
336 TRC: &ARM::SPRRegClass))
337 return true;
338
339 if (MI->isRegSequence() && usesRegClass(MO&: MI->getOperand(i: 1), TRC: &ARM::SPRRegClass))
340 return true;
341
342 return false;
343}
344
345// Looks through full copies to get the instruction that defines the input
346// operand for MI.
347MachineInstr *A15SDOptimizer::elideCopies(MachineInstr *MI) {
348 if (!MI->isFullCopy())
349 return MI;
350 if (!MI->getOperand(i: 1).getReg().isVirtual())
351 return nullptr;
352 MachineInstr *Def = MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg());
353 if (!Def)
354 return nullptr;
355 return elideCopies(MI: Def);
356}
357
358// Look through full copies and PHIs to get the set of non-copy MachineInstrs
359// that can produce MI.
360void A15SDOptimizer::elideCopiesAndPHIs(MachineInstr *MI,
361 SmallVectorImpl<MachineInstr*> &Outs) {
362 // Looking through PHIs may create loops so we need to track what
363 // instructions we have visited before.
364 std::set<MachineInstr *> Reached;
365 SmallVector<MachineInstr *, 8> Front;
366 Front.push_back(Elt: MI);
367 while (Front.size() != 0) {
368 MI = Front.pop_back_val();
369
370 // If we have already explored this MachineInstr, ignore it.
371 if (!Reached.insert(x: MI).second)
372 continue;
373 if (MI->isPHI()) {
374 for (unsigned I = 1, E = MI->getNumOperands(); I != E; I += 2) {
375 Register Reg = MI->getOperand(i: I).getReg();
376 if (!Reg.isVirtual()) {
377 continue;
378 }
379 MachineInstr *NewMI = MRI->getVRegDef(Reg);
380 if (!NewMI)
381 continue;
382 Front.push_back(Elt: NewMI);
383 }
384 } else if (MI->isFullCopy()) {
385 if (!MI->getOperand(i: 1).getReg().isVirtual())
386 continue;
387 MachineInstr *NewMI = MRI->getVRegDef(Reg: MI->getOperand(i: 1).getReg());
388 if (!NewMI)
389 continue;
390 Front.push_back(Elt: NewMI);
391 } else {
392 LLVM_DEBUG(dbgs() << "Found partial copy" << *MI << "\n");
393 Outs.push_back(Elt: MI);
394 }
395 }
396}
397
398// Return the DPR virtual registers that are read by this machine instruction
399// (if any).
400SmallVector<unsigned, 8> A15SDOptimizer::getReadDPRs(MachineInstr *MI) {
401 if (MI->isCopyLike() || MI->isInsertSubreg() || MI->isRegSequence() ||
402 MI->isKill())
403 return SmallVector<unsigned, 8>();
404
405 SmallVector<unsigned, 8> Defs;
406 for (MachineOperand &MO : MI->operands()) {
407 if (!MO.isReg() || !MO.isUse())
408 continue;
409 if (!usesRegClass(MO, TRC: &ARM::DPRRegClass) &&
410 !usesRegClass(MO, TRC: &ARM::QPRRegClass) &&
411 !usesRegClass(MO, TRC: &ARM::DPairRegClass)) // Treat DPair as QPR
412 continue;
413
414 Defs.push_back(Elt: MO.getReg());
415 }
416 return Defs;
417}
418
419// Creates a DPR register from an SPR one by using a VDUP.
420unsigned A15SDOptimizer::createDupLane(MachineBasicBlock &MBB,
421 MachineBasicBlock::iterator InsertBefore,
422 const DebugLoc &DL, unsigned Reg,
423 unsigned Lane, bool QPR) {
424 Register Out =
425 MRI->createVirtualRegister(RegClass: QPR ? &ARM::QPRRegClass : &ARM::DPRRegClass);
426 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DL,
427 MCID: TII->get(Opcode: QPR ? ARM::VDUPLN32q : ARM::VDUPLN32d), DestReg: Out)
428 .addReg(RegNo: Reg)
429 .addImm(Val: Lane)
430 .add(MOs: predOps(Pred: ARMCC::AL));
431
432 return Out;
433}
434
435// Creates a SPR register from a DPR by copying the value in lane 0.
436unsigned A15SDOptimizer::createExtractSubreg(
437 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
438 const DebugLoc &DL, unsigned DReg, unsigned Lane,
439 const TargetRegisterClass *TRC) {
440 Register Out = MRI->createVirtualRegister(RegClass: TRC);
441 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: Out)
442 .addReg(RegNo: DReg, Flags: {}, SubReg: Lane);
443
444 return Out;
445}
446
447// Takes two SPR registers and creates a DPR by using a REG_SEQUENCE.
448unsigned A15SDOptimizer::createRegSequence(
449 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
450 const DebugLoc &DL, unsigned Reg1, unsigned Reg2) {
451 Register Out = MRI->createVirtualRegister(RegClass: &ARM::QPRRegClass);
452 BuildMI(BB&: MBB,
453 I: InsertBefore,
454 MIMD: DL,
455 MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Out)
456 .addReg(RegNo: Reg1)
457 .addImm(Val: ARM::dsub_0)
458 .addReg(RegNo: Reg2)
459 .addImm(Val: ARM::dsub_1);
460 return Out;
461}
462
463// Takes two DPR registers that have previously been VDUPed (Ssub0 and Ssub1)
464// and merges them into one DPR register.
465unsigned A15SDOptimizer::createVExt(MachineBasicBlock &MBB,
466 MachineBasicBlock::iterator InsertBefore,
467 const DebugLoc &DL, unsigned Ssub0,
468 unsigned Ssub1) {
469 Register Out = MRI->createVirtualRegister(RegClass: &ARM::DPRRegClass);
470 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: ARM::VEXTd32), DestReg: Out)
471 .addReg(RegNo: Ssub0)
472 .addReg(RegNo: Ssub1)
473 .addImm(Val: 1)
474 .add(MOs: predOps(Pred: ARMCC::AL));
475 return Out;
476}
477
478unsigned A15SDOptimizer::createInsertSubreg(
479 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
480 const DebugLoc &DL, unsigned DReg, unsigned Lane, unsigned ToInsert) {
481 Register Out = MRI->createVirtualRegister(RegClass: &ARM::DPR_VFP2RegClass);
482 BuildMI(BB&: MBB,
483 I: InsertBefore,
484 MIMD: DL,
485 MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Out)
486 .addReg(RegNo: DReg)
487 .addReg(RegNo: ToInsert)
488 .addImm(Val: Lane);
489
490 return Out;
491}
492
493unsigned
494A15SDOptimizer::createImplicitDef(MachineBasicBlock &MBB,
495 MachineBasicBlock::iterator InsertBefore,
496 const DebugLoc &DL) {
497 Register Out = MRI->createVirtualRegister(RegClass: &ARM::DPRRegClass);
498 BuildMI(BB&: MBB,
499 I: InsertBefore,
500 MIMD: DL,
501 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Out);
502 return Out;
503}
504
505// This function inserts instructions in order to optimize interactions between
506// SPR registers and DPR/QPR registers. It does so by performing VDUPs on all
507// lanes, and the using VEXT instructions to recompose the result.
508unsigned
509A15SDOptimizer::optimizeAllLanesPattern(MachineInstr *MI, unsigned Reg) {
510 MachineBasicBlock::iterator InsertPt(MI);
511 DebugLoc DL = MI->getDebugLoc();
512 MachineBasicBlock &MBB = *MI->getParent();
513 InsertPt++;
514 unsigned Out;
515
516 // DPair has the same length as QPR and also has two DPRs as subreg.
517 // Treat DPair as QPR.
518 if (MRI->getRegClass(Reg)->hasSuperClassEq(RC: &ARM::QPRRegClass) ||
519 MRI->getRegClass(Reg)->hasSuperClassEq(RC: &ARM::DPairRegClass)) {
520 unsigned DSub0 = createExtractSubreg(MBB, InsertBefore: InsertPt, DL, DReg: Reg,
521 Lane: ARM::dsub_0, TRC: &ARM::DPRRegClass);
522 unsigned DSub1 = createExtractSubreg(MBB, InsertBefore: InsertPt, DL, DReg: Reg,
523 Lane: ARM::dsub_1, TRC: &ARM::DPRRegClass);
524
525 unsigned Out1 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg: DSub0, Lane: 0);
526 unsigned Out2 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg: DSub0, Lane: 1);
527 Out = createVExt(MBB, InsertBefore: InsertPt, DL, Ssub0: Out1, Ssub1: Out2);
528
529 unsigned Out3 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg: DSub1, Lane: 0);
530 unsigned Out4 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg: DSub1, Lane: 1);
531 Out2 = createVExt(MBB, InsertBefore: InsertPt, DL, Ssub0: Out3, Ssub1: Out4);
532
533 Out = createRegSequence(MBB, InsertBefore: InsertPt, DL, Reg1: Out, Reg2: Out2);
534
535 } else if (MRI->getRegClass(Reg)->hasSuperClassEq(RC: &ARM::DPRRegClass)) {
536 unsigned Out1 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg, Lane: 0);
537 unsigned Out2 = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg, Lane: 1);
538 Out = createVExt(MBB, InsertBefore: InsertPt, DL, Ssub0: Out1, Ssub1: Out2);
539
540 } else {
541 assert(MRI->getRegClass(Reg)->hasSuperClassEq(&ARM::SPRRegClass) &&
542 "Found unexpected regclass!");
543
544 unsigned PrefLane = getPrefSPRLane(SReg: Reg);
545 unsigned Lane;
546 switch (PrefLane) {
547 case ARM::ssub_0: Lane = 0; break;
548 case ARM::ssub_1: Lane = 1; break;
549 default: llvm_unreachable("Unknown preferred lane!");
550 }
551
552 // Treat DPair as QPR
553 bool UsesQPR = usesRegClass(MO&: MI->getOperand(i: 0), TRC: &ARM::QPRRegClass) ||
554 usesRegClass(MO&: MI->getOperand(i: 0), TRC: &ARM::DPairRegClass);
555
556 Out = createImplicitDef(MBB, InsertBefore: InsertPt, DL);
557 Out = createInsertSubreg(MBB, InsertBefore: InsertPt, DL, DReg: Out, Lane: PrefLane, ToInsert: Reg);
558 Out = createDupLane(MBB, InsertBefore: InsertPt, DL, Reg: Out, Lane, QPR: UsesQPR);
559 eraseInstrWithNoUses(MI);
560 }
561 return Out;
562}
563
564bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) {
565 // We look for instructions that write S registers that are then read as
566 // D/Q registers. These can only be caused by COPY, INSERT_SUBREG and
567 // REG_SEQUENCE pseudos that insert an SPR value into a DPR register or
568 // merge two SPR values to form a DPR register. In order avoid false
569 // positives we make sure that there is an SPR producer so we look past
570 // COPY and PHI nodes to find it.
571 //
572 // The best code pattern for when an SPR producer is going to be used by a
573 // DPR or QPR consumer depends on whether the other lanes of the
574 // corresponding DPR/QPR are currently defined.
575 //
576 // We can handle these efficiently, depending on the type of
577 // pseudo-instruction that is producing the pattern
578 //
579 // * COPY: * VDUP all lanes and merge the results together
580 // using VEXTs.
581 //
582 // * INSERT_SUBREG: * If the SPR value was originally in another DPR/QPR
583 // lane, and the other lane(s) of the DPR/QPR register
584 // that we are inserting in are undefined, use the
585 // original DPR/QPR value.
586 // * Otherwise, fall back on the same strategy as COPY.
587 //
588 // * REG_SEQUENCE: * If all except one of the input operands are
589 // IMPLICIT_DEFs, insert the VDUP pattern for just the
590 // defined input operand
591 // * Otherwise, fall back on the same strategy as COPY.
592 //
593
594 // First, get all the reads of D-registers done by this instruction.
595 SmallVector<unsigned, 8> Defs = getReadDPRs(MI);
596 bool Modified = false;
597
598 for (unsigned I : Defs) {
599 // Follow the def-use chain for this DPR through COPYs, and also through
600 // PHIs (which are essentially multi-way COPYs). It is because of PHIs that
601 // we can end up with multiple defs of this DPR.
602
603 SmallVector<MachineInstr *, 8> DefSrcs;
604 if (!Register::isVirtualRegister(Reg: I))
605 continue;
606 MachineInstr *Def = MRI->getVRegDef(Reg: I);
607 if (!Def)
608 continue;
609
610 elideCopiesAndPHIs(MI: Def, Outs&: DefSrcs);
611
612 for (MachineInstr *MI : DefSrcs) {
613 // If we've already analyzed and replaced this operand, don't do
614 // anything.
615 if (Replacements.find(x: MI) != Replacements.end())
616 continue;
617
618 // Now, work out if the instruction causes a SPR->DPR dependency.
619 if (!hasPartialWrite(MI))
620 continue;
621
622 // Collect all the uses of this MI's DPR def for updating later.
623 Register DPRDefReg = MI->getOperand(i: 0).getReg();
624 SmallVector<MachineOperand *, 8> Uses(
625 llvm::make_pointer_range(Range: MRI->use_operands(Reg: DPRDefReg)));
626
627 // We can optimize this.
628 unsigned NewReg = optimizeSDPattern(MI);
629
630 if (NewReg != 0) {
631 Modified = true;
632 for (MachineOperand *Use : Uses) {
633 // Make sure to constrain the register class of the new register to
634 // match what we're replacing. Otherwise we can optimize a DPR_VFP2
635 // reference into a plain DPR, and that will end poorly. NewReg is
636 // always virtual here, so there will always be a matching subclass
637 // to find.
638 MRI->constrainRegClass(Reg: NewReg, RC: MRI->getRegClass(Reg: Use->getReg()));
639
640 LLVM_DEBUG(dbgs() << "Replacing operand " << *Use << " with "
641 << printReg(NewReg) << "\n");
642 Use->substVirtReg(Reg: NewReg, SubIdx: 0, *TRI);
643 }
644 }
645 Replacements[MI] = NewReg;
646 }
647 }
648 return Modified;
649}
650
651bool A15SDOptimizer::runOnMachineFunction(MachineFunction &Fn) {
652 if (skipFunction(F: Fn.getFunction()))
653 return false;
654
655 const ARMSubtarget &STI = Fn.getSubtarget<ARMSubtarget>();
656 // Since the A15SDOptimizer pass can insert VDUP instructions, it can only be
657 // enabled when NEON is available.
658 if (!(STI.useSplatVFPToNeon() && STI.hasNEON()))
659 return false;
660
661 TII = STI.getInstrInfo();
662 TRI = STI.getRegisterInfo();
663 MRI = &Fn.getRegInfo();
664 bool Modified = false;
665
666 LLVM_DEBUG(dbgs() << "Running on function " << Fn.getName() << "\n");
667
668 DeadInstr.clear();
669 Replacements.clear();
670
671 for (MachineBasicBlock &MBB : Fn) {
672 for (MachineInstr &MI : MBB) {
673 Modified |= runOnInstruction(MI: &MI);
674 }
675 }
676
677 for (MachineInstr *MI : DeadInstr) {
678 MI->eraseFromParent();
679 }
680
681 return Modified;
682}
683
684FunctionPass *llvm::createA15SDOptimizerPass() {
685 return new A15SDOptimizer();
686}
687