1//===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements a simple VLIW packetizer using DFA. The packetizer works on
10// machine basic blocks. For each instruction I in BB, the packetizer consults
11// the DFA to see if machine resources are available to execute I. If so, the
12// packetizer checks if I depends on any instruction J in the current packet.
13// If no dependency is found, I is added to current packet and machine resource
14// is marked as taken. If any dependency is found, a target API call is made to
15// prune the dependence.
16//
17//===----------------------------------------------------------------------===//
18
19#include "HexagonVLIWPacketizer.h"
20#include "Hexagon.h"
21#include "HexagonInstrInfo.h"
22#include "HexagonRegisterInfo.h"
23#include "HexagonSubtarget.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseSet.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/AliasAnalysis.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
31#include "llvm/CodeGen/MachineDominators.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineFunctionPass.h"
35#include "llvm/CodeGen/MachineInstr.h"
36#include "llvm/CodeGen/MachineInstrBundle.h"
37#include "llvm/CodeGen/MachineLoopInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/ScheduleDAG.h"
40#include "llvm/CodeGen/TargetRegisterInfo.h"
41#include "llvm/CodeGen/TargetSubtargetInfo.h"
42#include "llvm/IR/DebugLoc.h"
43#include "llvm/InitializePasses.h"
44#include "llvm/MC/MCInstrDesc.h"
45#include "llvm/Pass.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/raw_ostream.h"
50#include <cassert>
51#include <cstdint>
52#include <iterator>
53
54using namespace llvm;
55
56#define DEBUG_TYPE "packets"
57
58static cl::opt<bool>
59 DisablePacketizer("disable-packetizer", cl::Hidden,
60 cl::desc("Disable Hexagon packetizer pass"));
61
62static cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden,
63 cl::init(Val: true),
64 cl::desc("Allow slot1 store and slot0 load"));
65
66static cl::opt<bool> PacketizeVolatiles(
67 "hexagon-packetize-volatiles", cl::Hidden, cl::init(Val: true),
68 cl::desc("Allow non-solo packetization of volatile memory references"));
69
70static cl::opt<bool>
71 EnableGenAllInsnClass("enable-gen-insn", cl::Hidden,
72 cl::desc("Generate all instruction with TC"));
73
74static cl::opt<bool>
75 DisableVecDblNVStores("disable-vecdbl-nv-stores", cl::Hidden,
76 cl::desc("Disable vector double new-value-stores"));
77
78extern cl::opt<bool> ScheduleInlineAsm;
79
80namespace llvm {
81
82FunctionPass *createHexagonPacketizer(bool Minimal);
83void initializeHexagonPacketizerPass(PassRegistry&);
84
85} // end namespace llvm
86
87namespace {
88
89 class HexagonPacketizer : public MachineFunctionPass {
90 public:
91 static char ID;
92
93 HexagonPacketizer(bool Min = false)
94 : MachineFunctionPass(ID), Minimal(Min) {}
95
96 void getAnalysisUsage(AnalysisUsage &AU) const override {
97 AU.setPreservesCFG();
98 AU.addRequired<AAResultsWrapperPass>();
99 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
100 AU.addRequired<MachineDominatorTreeWrapperPass>();
101 AU.addRequired<MachineLoopInfoWrapperPass>();
102 AU.addPreserved<MachineDominatorTreeWrapperPass>();
103 AU.addPreserved<MachineLoopInfoWrapperPass>();
104 MachineFunctionPass::getAnalysisUsage(AU);
105 }
106
107 StringRef getPassName() const override { return "Hexagon Packetizer"; }
108 bool runOnMachineFunction(MachineFunction &Fn) override;
109
110 MachineFunctionProperties getRequiredProperties() const override {
111 return MachineFunctionProperties().set(
112 MachineFunctionProperties::Property::NoVRegs);
113 }
114
115 private:
116 const HexagonInstrInfo *HII = nullptr;
117 const HexagonRegisterInfo *HRI = nullptr;
118 const bool Minimal = false;
119 };
120
121} // end anonymous namespace
122
123char HexagonPacketizer::ID = 0;
124
125INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
126 "Hexagon Packetizer", false, false)
127INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
128INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
129INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
130INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
131INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",
132 "Hexagon Packetizer", false, false)
133
134HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
135 MachineLoopInfo &MLI, AAResults *AA,
136 const MachineBranchProbabilityInfo *MBPI, bool Minimal)
137 : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI),
138 Minimal(Minimal) {
139 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
140 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
141
142 addMutation(Mutation: std::make_unique<HexagonSubtarget::UsrOverflowMutation>());
143 addMutation(Mutation: std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
144 addMutation(Mutation: std::make_unique<HexagonSubtarget::BankConflictMutation>());
145}
146
147// Check if FirstI modifies a register that SecondI reads.
148static bool hasWriteToReadDep(const MachineInstr &FirstI,
149 const MachineInstr &SecondI,
150 const TargetRegisterInfo *TRI) {
151 for (auto &MO : FirstI.operands()) {
152 if (!MO.isReg() || !MO.isDef())
153 continue;
154 Register R = MO.getReg();
155 if (SecondI.readsRegister(Reg: R, TRI))
156 return true;
157 }
158 return false;
159}
160
161
162static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI,
163 MachineBasicBlock::iterator BundleIt, bool Before) {
164 MachineBasicBlock::instr_iterator InsertPt;
165 if (Before)
166 InsertPt = BundleIt.getInstrIterator();
167 else
168 InsertPt = std::next(x: BundleIt).getInstrIterator();
169
170 MachineBasicBlock &B = *MI.getParent();
171 // The instruction should at least be bundled with the preceding instruction
172 // (there will always be one, i.e. BUNDLE, if nothing else).
173 assert(MI.isBundledWithPred());
174 if (MI.isBundledWithSucc()) {
175 MI.clearFlag(Flag: MachineInstr::BundledSucc);
176 MI.clearFlag(Flag: MachineInstr::BundledPred);
177 } else {
178 // If it's not bundled with the successor (i.e. it is the last one
179 // in the bundle), then we can simply unbundle it from the predecessor,
180 // which will take care of updating the predecessor's flag.
181 MI.unbundleFromPred();
182 }
183 B.splice(Where: InsertPt, Other: &B, From: MI.getIterator());
184
185 // Get the size of the bundle without asserting.
186 MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
187 MachineBasicBlock::const_instr_iterator E = B.instr_end();
188 unsigned Size = 0;
189 for (++I; I != E && I->isBundledWithPred(); ++I)
190 ++Size;
191
192 // If there are still two or more instructions, then there is nothing
193 // else to be done.
194 if (Size > 1)
195 return BundleIt;
196
197 // Otherwise, extract the single instruction out and delete the bundle.
198 MachineBasicBlock::iterator NextIt = std::next(x: BundleIt);
199 MachineInstr &SingleI = *BundleIt->getNextNode();
200 SingleI.unbundleFromPred();
201 assert(!SingleI.isBundledWithSucc());
202 BundleIt->eraseFromParent();
203 return NextIt;
204}
205
206bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
207 // FIXME: This pass causes verification failures.
208 MF.getProperties().set(
209 MachineFunctionProperties::Property::FailsVerification);
210
211 auto &HST = MF.getSubtarget<HexagonSubtarget>();
212 HII = HST.getInstrInfo();
213 HRI = HST.getRegisterInfo();
214 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
215 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
216 auto *MBPI =
217 &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
218
219 if (EnableGenAllInsnClass)
220 HII->genAllInsnTimingClasses(MF);
221
222 // Instantiate the packetizer.
223 bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() ||
224 skipFunction(F: MF.getFunction());
225 HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI, MinOnly);
226
227 // DFA state table should not be empty.
228 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
229
230 // Loop over all basic blocks and remove KILL pseudo-instructions
231 // These instructions confuse the dependence analysis. Consider:
232 // D0 = ... (Insn 0)
233 // R0 = KILL R0, D0 (Insn 1)
234 // R0 = ... (Insn 2)
235 // Here, Insn 1 will result in the dependence graph not emitting an output
236 // dependence between Insn 0 and Insn 2. This can lead to incorrect
237 // packetization
238 for (MachineBasicBlock &MB : MF) {
239 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MB))
240 if (MI.isKill())
241 MB.erase(I: &MI);
242 }
243
244 // TinyCore with Duplexes: Translate to big-instructions.
245 if (HST.isTinyCoreWithDuplex())
246 HII->translateInstrsForDup(MF, ToBigInstrs: true);
247
248 // Loop over all of the basic blocks.
249 for (auto &MB : MF) {
250 auto Begin = MB.begin(), End = MB.end();
251 while (Begin != End) {
252 // Find the first non-boundary starting from the end of the last
253 // scheduling region.
254 MachineBasicBlock::iterator RB = Begin;
255 while (RB != End && HII->isSchedulingBoundary(MI: *RB, MBB: &MB, MF))
256 ++RB;
257 // Find the first boundary starting from the beginning of the new
258 // region.
259 MachineBasicBlock::iterator RE = RB;
260 while (RE != End && !HII->isSchedulingBoundary(MI: *RE, MBB: &MB, MF))
261 ++RE;
262 // Add the scheduling boundary if it's not block end.
263 if (RE != End)
264 ++RE;
265 // If RB == End, then RE == End.
266 if (RB != End)
267 Packetizer.PacketizeMIs(MBB: &MB, BeginItr: RB, EndItr: RE);
268
269 Begin = RE;
270 }
271 }
272
273 // TinyCore with Duplexes: Translate to tiny-instructions.
274 if (HST.isTinyCoreWithDuplex())
275 HII->translateInstrsForDup(MF, ToBigInstrs: false);
276
277 Packetizer.unpacketizeSoloInstrs(MF);
278 return true;
279}
280
281// Reserve resources for a constant extender. Trigger an assertion if the
282// reservation fails.
283void HexagonPacketizerList::reserveResourcesForConstExt() {
284 if (!tryAllocateResourcesForConstExt(Reserve: true))
285 llvm_unreachable("Resources not available");
286}
287
288bool HexagonPacketizerList::canReserveResourcesForConstExt() {
289 return tryAllocateResourcesForConstExt(Reserve: false);
290}
291
292// Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
293// return true, otherwise, return false.
294bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
295 auto *ExtMI = MF.CreateMachineInstr(MCID: HII->get(Opcode: Hexagon::A4_ext), DL: DebugLoc());
296 bool Avail = ResourceTracker->canReserveResources(MI&: *ExtMI);
297 if (Reserve && Avail)
298 ResourceTracker->reserveResources(MI&: *ExtMI);
299 MF.deleteMachineInstr(MI: ExtMI);
300 return Avail;
301}
302
303bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI,
304 SDep::Kind DepType, unsigned DepReg) {
305 // Check for LR dependence.
306 if (DepReg == HRI->getRARegister())
307 return true;
308
309 if (HII->isDeallocRet(MI))
310 if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
311 return true;
312
313 // Call-like instructions can be packetized with preceding instructions
314 // that define registers implicitly used or modified by the call. Explicit
315 // uses are still prohibited, as in the case of indirect calls:
316 // r0 = ...
317 // J2_jumpr r0
318 if (DepType == SDep::Data) {
319 for (const MachineOperand &MO : MI.operands())
320 if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())
321 return true;
322 }
323
324 return false;
325}
326
327static bool isRegDependence(const SDep::Kind DepType) {
328 return DepType == SDep::Data || DepType == SDep::Anti ||
329 DepType == SDep::Output;
330}
331
332static bool isDirectJump(const MachineInstr &MI) {
333 return MI.getOpcode() == Hexagon::J2_jump;
334}
335
336static bool isSchedBarrier(const MachineInstr &MI) {
337 switch (MI.getOpcode()) {
338 case Hexagon::Y2_barrier:
339 return true;
340 }
341 return false;
342}
343
344static bool isControlFlow(const MachineInstr &MI) {
345 return MI.getDesc().isTerminator() || MI.getDesc().isCall();
346}
347
348/// Returns true if the instruction modifies a callee-saved register.
349static bool doesModifyCalleeSavedReg(const MachineInstr &MI,
350 const TargetRegisterInfo *TRI) {
351 const MachineFunction &MF = *MI.getParent()->getParent();
352 for (auto *CSR = TRI->getCalleeSavedRegs(MF: &MF); CSR && *CSR; ++CSR)
353 if (MI.modifiesRegister(Reg: *CSR, TRI))
354 return true;
355 return false;
356}
357
358// Returns true if an instruction can be promoted to .new predicate or
359// new-value store.
360bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI,
361 const TargetRegisterClass *NewRC) {
362 // Vector stores can be predicated, and can be new-value stores, but
363 // they cannot be predicated on a .new predicate value.
364 if (NewRC == &Hexagon::PredRegsRegClass) {
365 if (HII->isHVXVec(MI) && MI.mayStore())
366 return false;
367 return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, MBPI: nullptr) > 0;
368 }
369 // If the class is not PredRegs, it could only apply to new-value stores.
370 return HII->mayBeNewStore(MI);
371}
372
373// Promote an instructiont to its .cur form.
374// At this time, we have already made a call to canPromoteToDotCur and made
375// sure that it can *indeed* be promoted.
376bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI,
377 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
378 const TargetRegisterClass* RC) {
379 assert(DepType == SDep::Data);
380 int CurOpcode = HII->getDotCurOp(MI);
381 MI.setDesc(HII->get(Opcode: CurOpcode));
382 return true;
383}
384
385void HexagonPacketizerList::cleanUpDotCur() {
386 MachineInstr *MI = nullptr;
387 for (auto *BI : CurrentPacketMIs) {
388 LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
389 if (HII->isDotCurInst(MI: *BI)) {
390 MI = BI;
391 continue;
392 }
393 if (MI) {
394 for (auto &MO : BI->operands())
395 if (MO.isReg() && MO.getReg() == MI->getOperand(i: 0).getReg())
396 return;
397 }
398 }
399 if (!MI)
400 return;
401 // We did not find a use of the CUR, so de-cur it.
402 MI->setDesc(HII->get(Opcode: HII->getNonDotCurOp(MI: *MI)));
403 LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
404}
405
406// Check to see if an instruction can be dot cur.
407bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
408 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
409 const TargetRegisterClass *RC) {
410 if (!HII->isHVXVec(MI))
411 return false;
412 if (!HII->isHVXVec(MI: *MII))
413 return false;
414
415 // Already a dot new instruction.
416 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
417 return false;
418
419 if (!HII->mayBeCurLoad(MI))
420 return false;
421
422 // The "cur value" cannot come from inline asm.
423 if (PacketSU->getInstr()->isInlineAsm())
424 return false;
425
426 // Make sure candidate instruction uses cur.
427 LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump();
428 dbgs() << "in packet\n";);
429 MachineInstr &MJ = *MII;
430 LLVM_DEBUG({
431 dbgs() << "Checking CUR against ";
432 MJ.dump();
433 });
434 Register DestReg = MI.getOperand(i: 0).getReg();
435 bool FoundMatch = false;
436 for (auto &MO : MJ.operands())
437 if (MO.isReg() && MO.getReg() == DestReg)
438 FoundMatch = true;
439 if (!FoundMatch)
440 return false;
441
442 // Check for existing uses of a vector register within the packet which
443 // would be affected by converting a vector load into .cur formt.
444 for (auto *BI : CurrentPacketMIs) {
445 LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););
446 if (BI->readsRegister(Reg: DepReg, TRI: MF.getSubtarget().getRegisterInfo()))
447 return false;
448 }
449
450 LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
451 // We can convert the opcode into a .cur.
452 return true;
453}
454
455// Promote an instruction to its .new form. At this time, we have already
456// made a call to canPromoteToDotNew and made sure that it can *indeed* be
457// promoted.
458bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI,
459 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
460 const TargetRegisterClass* RC) {
461 assert(DepType == SDep::Data);
462 int NewOpcode;
463 if (RC == &Hexagon::PredRegsRegClass)
464 NewOpcode = HII->getDotNewPredOp(MI, MBPI);
465 else
466 NewOpcode = HII->getDotNewOp(MI);
467 MI.setDesc(HII->get(Opcode: NewOpcode));
468 return true;
469}
470
471bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) {
472 int NewOpcode = HII->getDotOldOp(MI);
473 MI.setDesc(HII->get(Opcode: NewOpcode));
474 return true;
475}
476
477bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
478 unsigned Opc = MI.getOpcode();
479 switch (Opc) {
480 case Hexagon::S2_storerd_io:
481 case Hexagon::S2_storeri_io:
482 case Hexagon::S2_storerh_io:
483 case Hexagon::S2_storerb_io:
484 break;
485 default:
486 llvm_unreachable("Unexpected instruction");
487 }
488 unsigned FrameSize = MF.getFrameInfo().getStackSize();
489 MachineOperand &Off = MI.getOperand(i: 1);
490 int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
491 if (HII->isValidOffset(Opcode: Opc, Offset: NewOff, TRI: HRI)) {
492 Off.setImm(NewOff);
493 return true;
494 }
495 return false;
496}
497
498void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) {
499 unsigned Opc = MI.getOpcode();
500 switch (Opc) {
501 case Hexagon::S2_storerd_io:
502 case Hexagon::S2_storeri_io:
503 case Hexagon::S2_storerh_io:
504 case Hexagon::S2_storerb_io:
505 break;
506 default:
507 llvm_unreachable("Unexpected instruction");
508 }
509 unsigned FrameSize = MF.getFrameInfo().getStackSize();
510 MachineOperand &Off = MI.getOperand(i: 1);
511 Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
512}
513
514/// Return true if we can update the offset in MI so that MI and MJ
515/// can be packetized together.
516bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) {
517 assert(SUI->getInstr() && SUJ->getInstr());
518 MachineInstr &MI = *SUI->getInstr();
519 MachineInstr &MJ = *SUJ->getInstr();
520
521 unsigned BPI, OPI;
522 if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BPI, OffsetPos&: OPI))
523 return false;
524 unsigned BPJ, OPJ;
525 if (!HII->getBaseAndOffsetPosition(MI: MJ, BasePos&: BPJ, OffsetPos&: OPJ))
526 return false;
527 Register Reg = MI.getOperand(i: BPI).getReg();
528 if (Reg != MJ.getOperand(i: BPJ).getReg())
529 return false;
530 // Make sure that the dependences do not restrict adding MI to the packet.
531 // That is, ignore anti dependences, and make sure the only data dependence
532 // involves the specific register.
533 for (const auto &PI : SUI->Preds)
534 if (PI.getKind() != SDep::Anti &&
535 (PI.getKind() != SDep::Data || PI.getReg() != Reg))
536 return false;
537 int Incr;
538 if (!HII->getIncrementValue(MI: MJ, Value&: Incr))
539 return false;
540
541 int64_t Offset = MI.getOperand(i: OPI).getImm();
542 if (!HII->isValidOffset(Opcode: MI.getOpcode(), Offset: Offset+Incr, TRI: HRI))
543 return false;
544
545 MI.getOperand(i: OPI).setImm(Offset + Incr);
546 ChangedOffset = Offset;
547 return true;
548}
549
550/// Undo the changed offset. This is needed if the instruction cannot be
551/// added to the current packet due to a different instruction.
552void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) {
553 unsigned BP, OP;
554 if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BP, OffsetPos&: OP))
555 llvm_unreachable("Unable to find base and offset operands.");
556 MI.getOperand(i: OP).setImm(ChangedOffset);
557}
558
559enum PredicateKind {
560 PK_False,
561 PK_True,
562 PK_Unknown
563};
564
565/// Returns true if an instruction is predicated on p0 and false if it's
566/// predicated on !p0.
567static PredicateKind getPredicateSense(const MachineInstr &MI,
568 const HexagonInstrInfo *HII) {
569 if (!HII->isPredicated(MI))
570 return PK_Unknown;
571 if (HII->isPredicatedTrue(MI))
572 return PK_True;
573 return PK_False;
574}
575
576static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI,
577 const HexagonInstrInfo *HII) {
578 assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
579#ifndef NDEBUG
580 // Post Increment means duplicates. Use dense map to find duplicates in the
581 // list. Caution: Densemap initializes with the minimum of 64 buckets,
582 // whereas there are at most 5 operands in the post increment.
583 DenseSet<unsigned> DefRegsSet;
584 for (auto &MO : MI.operands())
585 if (MO.isReg() && MO.isDef())
586 DefRegsSet.insert(MO.getReg());
587
588 for (auto &MO : MI.operands())
589 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
590 return MO;
591#else
592 if (MI.mayLoad()) {
593 const MachineOperand &Op1 = MI.getOperand(i: 1);
594 // The 2nd operand is always the post increment operand in load.
595 assert(Op1.isReg() && "Post increment operand has be to a register.");
596 return Op1;
597 }
598 if (MI.getDesc().mayStore()) {
599 const MachineOperand &Op0 = MI.getOperand(i: 0);
600 // The 1st operand is always the post increment operand in store.
601 assert(Op0.isReg() && "Post increment operand has be to a register.");
602 return Op0;
603 }
604#endif
605 // we should never come here.
606 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
607}
608
609// Get the value being stored.
610static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) {
611 // value being stored is always the last operand.
612 return MI.getOperand(i: MI.getNumOperands()-1);
613}
614
615static bool isLoadAbsSet(const MachineInstr &MI) {
616 unsigned Opc = MI.getOpcode();
617 switch (Opc) {
618 case Hexagon::L4_loadrd_ap:
619 case Hexagon::L4_loadrb_ap:
620 case Hexagon::L4_loadrh_ap:
621 case Hexagon::L4_loadrub_ap:
622 case Hexagon::L4_loadruh_ap:
623 case Hexagon::L4_loadri_ap:
624 return true;
625 }
626 return false;
627}
628
629static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) {
630 assert(isLoadAbsSet(MI));
631 return MI.getOperand(i: 1);
632}
633
634// Can be new value store?
635// Following restrictions are to be respected in convert a store into
636// a new value store.
637// 1. If an instruction uses auto-increment, its address register cannot
638// be a new-value register. Arch Spec 5.4.2.1
639// 2. If an instruction uses absolute-set addressing mode, its address
640// register cannot be a new-value register. Arch Spec 5.4.2.1.
641// 3. If an instruction produces a 64-bit result, its registers cannot be used
642// as new-value registers. Arch Spec 5.4.2.2.
643// 4. If the instruction that sets the new-value register is conditional, then
644// the instruction that uses the new-value register must also be conditional,
645// and both must always have their predicates evaluate identically.
646// Arch Spec 5.4.2.3.
647// 5. There is an implied restriction that a packet cannot have another store,
648// if there is a new value store in the packet. Corollary: if there is
649// already a store in a packet, there can not be a new value store.
650// Arch Spec: 3.4.4.2
651bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
652 const MachineInstr &PacketMI, unsigned DepReg) {
653 // Make sure we are looking at the store, that can be promoted.
654 if (!HII->mayBeNewStore(MI))
655 return false;
656
657 // Make sure there is dependency and can be new value'd.
658 const MachineOperand &Val = getStoreValueOperand(MI);
659 if (Val.isReg() && Val.getReg() != DepReg)
660 return false;
661
662 const MCInstrDesc& MCID = PacketMI.getDesc();
663
664 // First operand is always the result.
665 const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, OpNum: 0, TRI: HRI, MF);
666 // Double regs can not feed into new value store: PRM section: 5.4.2.2.
667 if (PacketRC == &Hexagon::DoubleRegsRegClass)
668 return false;
669
670 // New-value stores are of class NV (slot 0), dual stores require class ST
671 // in slot 0 (PRM 5.5).
672 for (auto *I : CurrentPacketMIs) {
673 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
674 if (PacketSU->getInstr()->mayStore())
675 return false;
676 }
677
678 // Make sure it's NOT the post increment register that we are going to
679 // new value.
680 if (HII->isPostIncrement(MI) &&
681 getPostIncrementOperand(MI, HII).getReg() == DepReg) {
682 return false;
683 }
684
685 if (HII->isPostIncrement(MI: PacketMI) && PacketMI.mayLoad() &&
686 getPostIncrementOperand(MI: PacketMI, HII).getReg() == DepReg) {
687 // If source is post_inc, or absolute-set addressing, it can not feed
688 // into new value store
689 // r3 = memw(r2++#4)
690 // memw(r30 + #-1404) = r2.new -> can not be new value store
691 // arch spec section: 5.4.2.1.
692 return false;
693 }
694
695 if (isLoadAbsSet(MI: PacketMI) && getAbsSetOperand(MI: PacketMI).getReg() == DepReg)
696 return false;
697
698 // If the source that feeds the store is predicated, new value store must
699 // also be predicated.
700 if (HII->isPredicated(MI: PacketMI)) {
701 if (!HII->isPredicated(MI))
702 return false;
703
704 // Check to make sure that they both will have their predicates
705 // evaluate identically.
706 unsigned predRegNumSrc = 0;
707 unsigned predRegNumDst = 0;
708 const TargetRegisterClass* predRegClass = nullptr;
709
710 // Get predicate register used in the source instruction.
711 for (auto &MO : PacketMI.operands()) {
712 if (!MO.isReg())
713 continue;
714 predRegNumSrc = MO.getReg();
715 predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumSrc);
716 if (predRegClass == &Hexagon::PredRegsRegClass)
717 break;
718 }
719 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
720 "predicate register not found in a predicated PacketMI instruction");
721
722 // Get predicate register used in new-value store instruction.
723 for (auto &MO : MI.operands()) {
724 if (!MO.isReg())
725 continue;
726 predRegNumDst = MO.getReg();
727 predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumDst);
728 if (predRegClass == &Hexagon::PredRegsRegClass)
729 break;
730 }
731 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
732 "predicate register not found in a predicated MI instruction");
733
734 // New-value register producer and user (store) need to satisfy these
735 // constraints:
736 // 1) Both instructions should be predicated on the same register.
737 // 2) If producer of the new-value register is .new predicated then store
738 // should also be .new predicated and if producer is not .new predicated
739 // then store should not be .new predicated.
740 // 3) Both new-value register producer and user should have same predicate
741 // sense, i.e, either both should be negated or both should be non-negated.
742 if (predRegNumDst != predRegNumSrc ||
743 HII->isDotNewInst(MI: PacketMI) != HII->isDotNewInst(MI) ||
744 getPredicateSense(MI, HII) != getPredicateSense(MI: PacketMI, HII))
745 return false;
746 }
747
748 // Make sure that other than the new-value register no other store instruction
749 // register has been modified in the same packet. Predicate registers can be
750 // modified by they should not be modified between the producer and the store
751 // instruction as it will make them both conditional on different values.
752 // We already know this to be true for all the instructions before and
753 // including PacketMI. Howerver, we need to perform the check for the
754 // remaining instructions in the packet.
755
756 unsigned StartCheck = 0;
757
758 for (auto *I : CurrentPacketMIs) {
759 SUnit *TempSU = MIToSUnit.find(x: I)->second;
760 MachineInstr &TempMI = *TempSU->getInstr();
761
762 // Following condition is true for all the instructions until PacketMI is
763 // reached (StartCheck is set to 0 before the for loop).
764 // StartCheck flag is 1 for all the instructions after PacketMI.
765 if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
766 continue; // encountering PacketMI.
767
768 StartCheck = 1;
769 if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
770 continue;
771
772 for (auto &MO : MI.operands())
773 if (MO.isReg() && TempSU->getInstr()->modifiesRegister(Reg: MO.getReg(), TRI: HRI))
774 return false;
775 }
776
777 // Make sure that for non-POST_INC stores:
778 // 1. The only use of reg is DepReg and no other registers.
779 // This handles base+index registers.
780 // The following store can not be dot new.
781 // Eg. r0 = add(r0, #3)
782 // memw(r1+r0<<#2) = r0
783 if (!HII->isPostIncrement(MI)) {
784 for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
785 const MachineOperand &MO = MI.getOperand(i: opNum);
786 if (MO.isReg() && MO.getReg() == DepReg)
787 return false;
788 }
789 }
790
791 // If data definition is because of implicit definition of the register,
792 // do not newify the store. Eg.
793 // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12
794 // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343]
795 for (auto &MO : PacketMI.operands()) {
796 if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg))
797 return false;
798 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
799 continue;
800 Register R = MO.getReg();
801 if (R == DepReg || HRI->isSuperRegister(RegA: DepReg, RegB: R))
802 return false;
803 }
804
805 // Handle imp-use of super reg case. There is a target independent side
806 // change that should prevent this situation but I am handling it for
807 // just-in-case. For example, we cannot newify R2 in the following case:
808 // %r3 = A2_tfrsi 0;
809 // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1;
810 for (auto &MO : MI.operands()) {
811 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
812 return false;
813 }
814
815 // Can be dot new store.
816 return true;
817}
818
819// Can this MI to promoted to either new value store or new value jump.
820bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI,
821 const SUnit *PacketSU, unsigned DepReg,
822 MachineBasicBlock::iterator &MII) {
823 if (!HII->mayBeNewStore(MI))
824 return false;
825
826 // Check to see the store can be new value'ed.
827 MachineInstr &PacketMI = *PacketSU->getInstr();
828 if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
829 return true;
830
831 // Check to see the compare/jump can be new value'ed.
832 // This is done as a pass on its own. Don't need to check it here.
833 return false;
834}
835
836static bool isImplicitDependency(const MachineInstr &I, bool CheckDef,
837 unsigned DepReg) {
838 for (auto &MO : I.operands()) {
839 if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg))
840 return true;
841 if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())
842 continue;
843 if (CheckDef == MO.isDef())
844 return true;
845 }
846 return false;
847}
848
849// Check to see if an instruction can be dot new.
850bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
851 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
852 const TargetRegisterClass* RC) {
853 // Already a dot new instruction.
854 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
855 return false;
856
857 if (!isNewifiable(MI, NewRC: RC))
858 return false;
859
860 const MachineInstr &PI = *PacketSU->getInstr();
861
862 // The "new value" cannot come from inline asm.
863 if (PI.isInlineAsm())
864 return false;
865
866 // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
867 // sense.
868 if (PI.isImplicitDef())
869 return false;
870
871 // If dependency is trough an implicitly defined register, we should not
872 // newify the use.
873 if (isImplicitDependency(I: PI, CheckDef: true, DepReg) ||
874 isImplicitDependency(I: MI, CheckDef: false, DepReg))
875 return false;
876
877 const MCInstrDesc& MCID = PI.getDesc();
878 const TargetRegisterClass *VecRC = HII->getRegClass(MCID, OpNum: 0, TRI: HRI, MF);
879 if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass)
880 return false;
881
882 // predicate .new
883 if (RC == &Hexagon::PredRegsRegClass)
884 return HII->predCanBeUsedAsDotNew(MI: PI, PredReg: DepReg);
885
886 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
887 return false;
888
889 // Create a dot new machine instruction to see if resources can be
890 // allocated. If not, bail out now.
891 int NewOpcode = (RC != &Hexagon::PredRegsRegClass) ? HII->getDotNewOp(MI) :
892 HII->getDotNewPredOp(MI, MBPI);
893 const MCInstrDesc &D = HII->get(Opcode: NewOpcode);
894 MachineInstr *NewMI = MF.CreateMachineInstr(MCID: D, DL: DebugLoc());
895 bool ResourcesAvailable = ResourceTracker->canReserveResources(MI&: *NewMI);
896 MF.deleteMachineInstr(MI: NewMI);
897 if (!ResourcesAvailable)
898 return false;
899
900 // New Value Store only. New Value Jump generated as a separate pass.
901 if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
902 return false;
903
904 return true;
905}
906
907// Go through the packet instructions and search for an anti dependency between
908// them and DepReg from MI. Consider this case:
909// Trying to add
910// a) %r1 = TFRI_cdNotPt %p3, 2
911// to this packet:
912// {
913// b) %p0 = C2_or killed %p3, killed %p0
914// c) %p3 = C2_tfrrp %r23
915// d) %r1 = C2_cmovenewit %p3, 4
916// }
917// The P3 from a) and d) will be complements after
918// a)'s P3 is converted to .new form
919// Anti-dep between c) and b) is irrelevant for this case
920bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI,
921 unsigned DepReg) {
922 SUnit *PacketSUDep = MIToSUnit.find(x: &MI)->second;
923
924 for (auto *I : CurrentPacketMIs) {
925 // We only care for dependencies to predicated instructions
926 if (!HII->isPredicated(MI: *I))
927 continue;
928
929 // Scheduling Unit for current insn in the packet
930 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
931
932 // Look at dependencies between current members of the packet and
933 // predicate defining instruction MI. Make sure that dependency is
934 // on the exact register we care about.
935 if (PacketSU->isSucc(N: PacketSUDep)) {
936 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
937 auto &Dep = PacketSU->Succs[i];
938 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
939 Dep.getReg() == DepReg)
940 return true;
941 }
942 }
943 }
944
945 return false;
946}
947
948/// Gets the predicate register of a predicated instruction.
949static unsigned getPredicatedRegister(MachineInstr &MI,
950 const HexagonInstrInfo *QII) {
951 /// We use the following rule: The first predicate register that is a use is
952 /// the predicate register of a predicated instruction.
953 assert(QII->isPredicated(MI) && "Must be predicated instruction");
954
955 for (auto &Op : MI.operands()) {
956 if (Op.isReg() && Op.getReg() && Op.isUse() &&
957 Hexagon::PredRegsRegClass.contains(Reg: Op.getReg()))
958 return Op.getReg();
959 }
960
961 llvm_unreachable("Unknown instruction operand layout");
962 return 0;
963}
964
965// Given two predicated instructions, this function detects whether
966// the predicates are complements.
967bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
968 MachineInstr &MI2) {
969 // If we don't know the predicate sense of the instructions bail out early, we
970 // need it later.
971 if (getPredicateSense(MI: MI1, HII) == PK_Unknown ||
972 getPredicateSense(MI: MI2, HII) == PK_Unknown)
973 return false;
974
975 // Scheduling unit for candidate.
976 SUnit *SU = MIToSUnit[&MI1];
977
978 // One corner case deals with the following scenario:
979 // Trying to add
980 // a) %r24 = A2_tfrt %p0, %r25
981 // to this packet:
982 // {
983 // b) %r25 = A2_tfrf %p0, %r24
984 // c) %p0 = C2_cmpeqi %r26, 1
985 // }
986 //
987 // On general check a) and b) are complements, but presence of c) will
988 // convert a) to .new form, and then it is not a complement.
989 // We attempt to detect it by analyzing existing dependencies in the packet.
990
991 // Analyze relationships between all existing members of the packet.
992 // Look for Anti dependecy on the same predicate reg as used in the
993 // candidate.
994 for (auto *I : CurrentPacketMIs) {
995 // Scheduling Unit for current insn in the packet.
996 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
997
998 // If this instruction in the packet is succeeded by the candidate...
999 if (PacketSU->isSucc(N: SU)) {
1000 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
1001 auto Dep = PacketSU->Succs[i];
1002 // The corner case exist when there is true data dependency between
1003 // candidate and one of current packet members, this dep is on
1004 // predicate reg, and there already exist anti dep on the same pred in
1005 // the packet.
1006 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
1007 Hexagon::PredRegsRegClass.contains(Reg: Dep.getReg())) {
1008 // Here I know that I is predicate setting instruction with true
1009 // data dep to candidate on the register we care about - c) in the
1010 // above example. Now I need to see if there is an anti dependency
1011 // from c) to any other instruction in the same packet on the pred
1012 // reg of interest.
1013 if (restrictingDepExistInPacket(MI&: *I, DepReg: Dep.getReg()))
1014 return false;
1015 }
1016 }
1017 }
1018 }
1019
1020 // If the above case does not apply, check regular complement condition.
1021 // Check that the predicate register is the same and that the predicate
1022 // sense is different We also need to differentiate .old vs. .new: !p0
1023 // is not complementary to p0.new.
1024 unsigned PReg1 = getPredicatedRegister(MI&: MI1, QII: HII);
1025 unsigned PReg2 = getPredicatedRegister(MI&: MI2, QII: HII);
1026 return PReg1 == PReg2 &&
1027 Hexagon::PredRegsRegClass.contains(Reg: PReg1) &&
1028 Hexagon::PredRegsRegClass.contains(Reg: PReg2) &&
1029 getPredicateSense(MI: MI1, HII) != getPredicateSense(MI: MI2, HII) &&
1030 HII->isDotNewInst(MI: MI1) == HII->isDotNewInst(MI: MI2);
1031}
1032
1033// Initialize packetizer flags.
1034void HexagonPacketizerList::initPacketizerState() {
1035 Dependence = false;
1036 PromotedToDotNew = false;
1037 GlueToNewValueJump = false;
1038 GlueAllocframeStore = false;
1039 FoundSequentialDependence = false;
1040 ChangedOffset = INT64_MAX;
1041}
1042
1043// Ignore bundling of pseudo instructions.
1044bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
1045 const MachineBasicBlock *) {
1046 if (MI.isDebugInstr())
1047 return true;
1048
1049 if (MI.isCFIInstruction())
1050 return false;
1051
1052 // We must print out inline assembly.
1053 if (MI.isInlineAsm())
1054 return false;
1055
1056 if (MI.isImplicitDef())
1057 return false;
1058
1059 // We check if MI has any functional units mapped to it. If it doesn't,
1060 // we ignore the instruction.
1061 const MCInstrDesc& TID = MI.getDesc();
1062 auto *IS = ResourceTracker->getInstrItins()->beginStage(ItinClassIndx: TID.getSchedClass());
1063 return !IS->getUnits();
1064}
1065
1066bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
1067 // Ensure any bundles created by gather packetize remain separate.
1068 if (MI.isBundle())
1069 return true;
1070
1071 if (MI.isEHLabel() || MI.isCFIInstruction())
1072 return true;
1073
1074 // Consider inline asm to not be a solo instruction by default.
1075 // Inline asm will be put in a packet temporarily, but then it will be
1076 // removed, and placed outside of the packet (before or after, depending
1077 // on dependencies). This is to reduce the impact of inline asm as a
1078 // "packet splitting" instruction.
1079 if (MI.isInlineAsm() && !ScheduleInlineAsm)
1080 return true;
1081
1082 if (isSchedBarrier(MI))
1083 return true;
1084
1085 if (HII->isSolo(MI))
1086 return true;
1087
1088 if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER ||
1089 MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT ||
1090 MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL)
1091 return true;
1092
1093 if (MI.getOpcode() == Hexagon::A2_nop)
1094 return true;
1095
1096 return false;
1097}
1098
1099// Quick check if instructions MI and MJ cannot coexist in the same packet.
1100// Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
1101// but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1102// For full test call this function twice:
1103// cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
1104// Doing the test only one way saves the amount of code in this function,
1105// since every test would need to be repeated with the MI and MJ reversed.
1106static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
1107 const HexagonInstrInfo &HII) {
1108 const MachineFunction *MF = MI.getParent()->getParent();
1109 if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() &&
1110 HII.isHVXMemWithAIndirect(I: MI, J: MJ))
1111 return true;
1112
1113 // Don't allow a store and an instruction that must be in slot0 and
1114 // doesn't allow a slot1 instruction.
1115 if (MI.mayStore() && HII.isRestrictNoSlot1Store(MI: MJ) && HII.isPureSlot0(MI: MJ))
1116 return true;
1117
1118 // An inline asm cannot be together with a branch, because we may not be
1119 // able to remove the asm out after packetizing (i.e. if the asm must be
1120 // moved past the bundle). Similarly, two asms cannot be together to avoid
1121 // complications when determining their relative order outside of a bundle.
1122 if (MI.isInlineAsm())
1123 return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
1124 MJ.isCall() || MJ.isTerminator();
1125
1126 // New-value stores cannot coexist with any other stores.
1127 if (HII.isNewValueStore(MI) && MJ.mayStore())
1128 return true;
1129
1130 switch (MI.getOpcode()) {
1131 case Hexagon::S2_storew_locked:
1132 case Hexagon::S4_stored_locked:
1133 case Hexagon::L2_loadw_locked:
1134 case Hexagon::L4_loadd_locked:
1135 case Hexagon::Y2_dccleana:
1136 case Hexagon::Y2_dccleaninva:
1137 case Hexagon::Y2_dcinva:
1138 case Hexagon::Y2_dczeroa:
1139 case Hexagon::Y4_l2fetch:
1140 case Hexagon::Y5_l2fetch: {
1141 // These instructions can only be grouped with ALU32 or non-floating-point
1142 // XTYPE instructions. Since there is no convenient way of identifying fp
1143 // XTYPE instructions, only allow grouping with ALU32 for now.
1144 unsigned TJ = HII.getType(MI: MJ);
1145 if (TJ != HexagonII::TypeALU32_2op &&
1146 TJ != HexagonII::TypeALU32_3op &&
1147 TJ != HexagonII::TypeALU32_ADDI)
1148 return true;
1149 break;
1150 }
1151 default:
1152 break;
1153 }
1154
1155 // "False" really means that the quick check failed to determine if
1156 // I and J cannot coexist.
1157 return false;
1158}
1159
1160// Full, symmetric check.
1161bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI,
1162 const MachineInstr &MJ) {
1163 return cannotCoexistAsymm(MI, MJ, HII: *HII) || cannotCoexistAsymm(MI: MJ, MJ: MI, HII: *HII);
1164}
1165
1166void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
1167 for (auto &B : MF) {
1168 MachineBasicBlock::iterator BundleIt;
1169 for (MachineInstr &MI : llvm::make_early_inc_range(Range: B.instrs())) {
1170 if (MI.isBundle())
1171 BundleIt = MI.getIterator();
1172 if (!MI.isInsideBundle())
1173 continue;
1174
1175 // Decide on where to insert the instruction that we are pulling out.
1176 // Debug instructions always go before the bundle, but the placement of
1177 // INLINE_ASM depends on potential dependencies. By default, try to
1178 // put it before the bundle, but if the asm writes to a register that
1179 // other instructions in the bundle read, then we need to place it
1180 // after the bundle (to preserve the bundle semantics).
1181 bool InsertBeforeBundle;
1182 if (MI.isInlineAsm())
1183 InsertBeforeBundle = !hasWriteToReadDep(FirstI: MI, SecondI: *BundleIt, TRI: HRI);
1184 else if (MI.isDebugInstr())
1185 InsertBeforeBundle = true;
1186 else
1187 continue;
1188
1189 BundleIt = moveInstrOut(MI, BundleIt, Before: InsertBeforeBundle);
1190 }
1191 }
1192}
1193
1194// Check if a given instruction is of class "system".
1195static bool isSystemInstr(const MachineInstr &MI) {
1196 unsigned Opc = MI.getOpcode();
1197 switch (Opc) {
1198 case Hexagon::Y2_barrier:
1199 case Hexagon::Y2_dcfetchbo:
1200 case Hexagon::Y4_l2fetch:
1201 case Hexagon::Y5_l2fetch:
1202 return true;
1203 }
1204 return false;
1205}
1206
1207bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I,
1208 const MachineInstr &J) {
1209 // The dependence graph may not include edges between dead definitions,
1210 // so without extra checks, we could end up packetizing two instruction
1211 // defining the same (dead) register.
1212 if (I.isCall() || J.isCall())
1213 return false;
1214 if (HII->isPredicated(MI: I) || HII->isPredicated(MI: J))
1215 return false;
1216
1217 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
1218 for (auto &MO : I.operands()) {
1219 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1220 continue;
1221 DeadDefs[MO.getReg()] = true;
1222 }
1223
1224 for (auto &MO : J.operands()) {
1225 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1226 continue;
1227 Register R = MO.getReg();
1228 if (R != Hexagon::USR_OVF && DeadDefs[R])
1229 return true;
1230 }
1231 return false;
1232}
1233
1234bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I,
1235 const MachineInstr &J) {
1236 // A save callee-save register function call can only be in a packet
1237 // with instructions that don't write to the callee-save registers.
1238 if ((HII->isSaveCalleeSavedRegsCall(MI: I) &&
1239 doesModifyCalleeSavedReg(MI: J, TRI: HRI)) ||
1240 (HII->isSaveCalleeSavedRegsCall(MI: J) &&
1241 doesModifyCalleeSavedReg(MI: I, TRI: HRI)))
1242 return true;
1243
1244 // Two control flow instructions cannot go in the same packet.
1245 if (isControlFlow(MI: I) && isControlFlow(MI: J))
1246 return true;
1247
1248 // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1249 // contain a speculative indirect jump,
1250 // a new-value compare jump or a dealloc_return.
1251 auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
1252 if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
1253 return true;
1254 if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
1255 return true;
1256 return false;
1257 };
1258
1259 if (HII->isLoopN(MI: I) && isBadForLoopN(J))
1260 return true;
1261 if (HII->isLoopN(MI: J) && isBadForLoopN(I))
1262 return true;
1263
1264 // dealloc_return cannot appear in the same packet as a conditional or
1265 // unconditional jump.
1266 return HII->isDeallocRet(MI: I) &&
1267 (J.isBranch() || J.isCall() || J.isBarrier());
1268}
1269
1270bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I,
1271 const MachineInstr &J) {
1272 // Adding I to a packet that has J.
1273
1274 // Regmasks are not reflected in the scheduling dependency graph, so
1275 // we need to check them manually. This code assumes that regmasks only
1276 // occur on calls, and the problematic case is when we add an instruction
1277 // defining a register R to a packet that has a call that clobbers R via
1278 // a regmask. Those cannot be packetized together, because the call will
1279 // be executed last. That's also a reson why it is ok to add a call
1280 // clobbering R to a packet that defines R.
1281
1282 // Look for regmasks in J.
1283 for (const MachineOperand &OpJ : J.operands()) {
1284 if (!OpJ.isRegMask())
1285 continue;
1286 assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");
1287 for (const MachineOperand &OpI : I.operands()) {
1288 if (OpI.isReg()) {
1289 if (OpJ.clobbersPhysReg(PhysReg: OpI.getReg()))
1290 return true;
1291 } else if (OpI.isRegMask()) {
1292 // Both are regmasks. Assume that they intersect.
1293 return true;
1294 }
1295 }
1296 }
1297 return false;
1298}
1299
1300bool HexagonPacketizerList::hasDualStoreDependence(const MachineInstr &I,
1301 const MachineInstr &J) {
1302 bool SysI = isSystemInstr(MI: I), SysJ = isSystemInstr(MI: J);
1303 bool StoreI = I.mayStore(), StoreJ = J.mayStore();
1304 if ((SysI && StoreJ) || (SysJ && StoreI))
1305 return true;
1306
1307 if (StoreI && StoreJ) {
1308 if (HII->isNewValueInst(MI: J) || HII->isMemOp(MI: J) || HII->isMemOp(MI: I))
1309 return true;
1310 } else {
1311 // A memop cannot be in the same packet with another memop or a store.
1312 // Two stores can be together, but here I and J cannot both be stores.
1313 bool MopStI = HII->isMemOp(MI: I) || StoreI;
1314 bool MopStJ = HII->isMemOp(MI: J) || StoreJ;
1315 if (MopStI && MopStJ)
1316 return true;
1317 }
1318
1319 return (StoreJ && HII->isDeallocRet(MI: I)) || (StoreI && HII->isDeallocRet(MI: J));
1320}
1321
1322// SUI is the current instruction that is outside of the current packet.
1323// SUJ is the current instruction inside the current packet against which that
1324// SUI will be packetized.
1325bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
1326 assert(SUI->getInstr() && SUJ->getInstr());
1327 MachineInstr &I = *SUI->getInstr();
1328 MachineInstr &J = *SUJ->getInstr();
1329
1330 // Clear IgnoreDepMIs when Packet starts.
1331 if (CurrentPacketMIs.size() == 1)
1332 IgnoreDepMIs.clear();
1333
1334 MachineBasicBlock::iterator II = I.getIterator();
1335
1336 // Solo instructions cannot go in the packet.
1337 assert(!isSoloInstruction(I) && "Unexpected solo instr!");
1338
1339 if (cannotCoexist(MI: I, MJ: J))
1340 return false;
1341
1342 Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1343 if (Dependence)
1344 return false;
1345
1346 // Regmasks are not accounted for in the scheduling graph, so we need
1347 // to explicitly check for dependencies caused by them. They should only
1348 // appear on calls, so it's not too pessimistic to reject all regmask
1349 // dependencies.
1350 Dependence = hasRegMaskDependence(I, J);
1351 if (Dependence)
1352 return false;
1353
1354 // Dual-store does not allow second store, if the first store is not
1355 // in SLOT0. New value store, new value jump, dealloc_return and memop
1356 // always take SLOT0. Arch spec 3.4.4.2.
1357 Dependence = hasDualStoreDependence(I, J);
1358 if (Dependence)
1359 return false;
1360
1361 // If an instruction feeds new value jump, glue it.
1362 MachineBasicBlock::iterator NextMII = I.getIterator();
1363 ++NextMII;
1364 if (NextMII != I.getParent()->end() && HII->isNewValueJump(MI: *NextMII)) {
1365 MachineInstr &NextMI = *NextMII;
1366
1367 bool secondRegMatch = false;
1368 const MachineOperand &NOp0 = NextMI.getOperand(i: 0);
1369 const MachineOperand &NOp1 = NextMI.getOperand(i: 1);
1370
1371 if (NOp1.isReg() && I.getOperand(i: 0).getReg() == NOp1.getReg())
1372 secondRegMatch = true;
1373
1374 for (MachineInstr *PI : CurrentPacketMIs) {
1375 // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
1376 if (PI->isCall()) {
1377 Dependence = true;
1378 break;
1379 }
1380 // Validate:
1381 // 1. Packet does not have a store in it.
1382 // 2. If the first operand of the nvj is newified, and the second
1383 // operand is also a reg, it (second reg) is not defined in
1384 // the same packet.
1385 // 3. If the second operand of the nvj is newified, (which means
1386 // first operand is also a reg), first reg is not defined in
1387 // the same packet.
1388 if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
1389 HII->isLoopN(MI: *PI)) {
1390 Dependence = true;
1391 break;
1392 }
1393 // Check #2/#3.
1394 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
1395 if (OpR.isReg() && PI->modifiesRegister(Reg: OpR.getReg(), TRI: HRI)) {
1396 Dependence = true;
1397 break;
1398 }
1399 }
1400
1401 GlueToNewValueJump = true;
1402 if (Dependence)
1403 return false;
1404 }
1405
1406 // There no dependency between a prolog instruction and its successor.
1407 if (!SUJ->isSucc(N: SUI))
1408 return true;
1409
1410 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1411 if (FoundSequentialDependence)
1412 break;
1413
1414 if (SUJ->Succs[i].getSUnit() != SUI)
1415 continue;
1416
1417 SDep::Kind DepType = SUJ->Succs[i].getKind();
1418 // For direct calls:
1419 // Ignore register dependences for call instructions for packetization
1420 // purposes except for those due to r31 and predicate registers.
1421 //
1422 // For indirect calls:
1423 // Same as direct calls + check for true dependences to the register
1424 // used in the indirect call.
1425 //
1426 // We completely ignore Order dependences for call instructions.
1427 //
1428 // For returns:
1429 // Ignore register dependences for return instructions like jumpr,
1430 // dealloc return unless we have dependencies on the explicit uses
1431 // of the registers used by jumpr (like r31) or dealloc return
1432 // (like r29 or r30).
1433 unsigned DepReg = 0;
1434 const TargetRegisterClass *RC = nullptr;
1435 if (DepType == SDep::Data) {
1436 DepReg = SUJ->Succs[i].getReg();
1437 RC = HRI->getMinimalPhysRegClass(Reg: DepReg);
1438 }
1439
1440 if (I.isCall() || HII->isJumpR(MI: I) || I.isReturn() || HII->isTailCall(MI: I)) {
1441 if (!isRegDependence(DepType))
1442 continue;
1443 if (!isCallDependent(MI: I, DepType, DepReg: SUJ->Succs[i].getReg()))
1444 continue;
1445 }
1446
1447 if (DepType == SDep::Data) {
1448 if (canPromoteToDotCur(MI: J, PacketSU: SUJ, DepReg, MII&: II, RC))
1449 if (promoteToDotCur(MI&: J, DepType, MII&: II, RC))
1450 continue;
1451 }
1452
1453 // Data dpendence ok if we have load.cur.
1454 if (DepType == SDep::Data && HII->isDotCurInst(MI: J)) {
1455 if (HII->isHVXVec(MI: I))
1456 continue;
1457 }
1458
1459 // For instructions that can be promoted to dot-new, try to promote.
1460 if (DepType == SDep::Data) {
1461 if (canPromoteToDotNew(MI: I, PacketSU: SUJ, DepReg, MII&: II, RC)) {
1462 if (promoteToDotNew(MI&: I, DepType, MII&: II, RC)) {
1463 PromotedToDotNew = true;
1464 if (cannotCoexist(MI: I, MJ: J))
1465 FoundSequentialDependence = true;
1466 continue;
1467 }
1468 }
1469 if (HII->isNewValueJump(MI: I))
1470 continue;
1471 }
1472
1473 // For predicated instructions, if the predicates are complements then
1474 // there can be no dependence.
1475 if (HII->isPredicated(MI: I) && HII->isPredicated(MI: J) &&
1476 arePredicatesComplements(MI1&: I, MI2&: J)) {
1477 // Not always safe to do this translation.
1478 // DAG Builder attempts to reduce dependence edges using transitive
1479 // nature of dependencies. Here is an example:
1480 //
1481 // r0 = tfr_pt ... (1)
1482 // r0 = tfr_pf ... (2)
1483 // r0 = tfr_pt ... (3)
1484 //
1485 // There will be an output dependence between (1)->(2) and (2)->(3).
1486 // However, there is no dependence edge between (1)->(3). This results
1487 // in all 3 instructions going in the same packet. We ignore dependce
1488 // only once to avoid this situation.
1489 auto Itr = find(Range&: IgnoreDepMIs, Val: &J);
1490 if (Itr != IgnoreDepMIs.end()) {
1491 Dependence = true;
1492 return false;
1493 }
1494 IgnoreDepMIs.push_back(x: &I);
1495 continue;
1496 }
1497
1498 // Ignore Order dependences between unconditional direct branches
1499 // and non-control-flow instructions.
1500 if (isDirectJump(MI: I) && !J.isBranch() && !J.isCall() &&
1501 DepType == SDep::Order)
1502 continue;
1503
1504 // Ignore all dependences for jumps except for true and output
1505 // dependences.
1506 if (I.isConditionalBranch() && DepType != SDep::Data &&
1507 DepType != SDep::Output)
1508 continue;
1509
1510 if (DepType == SDep::Output) {
1511 FoundSequentialDependence = true;
1512 break;
1513 }
1514
1515 // For Order dependences:
1516 // 1. Volatile loads/stores can be packetized together, unless other
1517 // rules prevent is.
1518 // 2. Store followed by a load is not allowed.
1519 // 3. Store followed by a store is valid.
1520 // 4. Load followed by any memory operation is allowed.
1521 if (DepType == SDep::Order) {
1522 if (!PacketizeVolatiles) {
1523 bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
1524 if (OrdRefs) {
1525 FoundSequentialDependence = true;
1526 break;
1527 }
1528 }
1529 // J is first, I is second.
1530 bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
1531 bool LoadI = I.mayLoad(), StoreI = I.mayStore();
1532 bool NVStoreJ = HII->isNewValueStore(MI: J);
1533 bool NVStoreI = HII->isNewValueStore(MI: I);
1534 bool IsVecJ = HII->isHVXVec(MI: J);
1535 bool IsVecI = HII->isHVXVec(MI: I);
1536
1537 // Don't reorder the loads if there is an order dependence. This would
1538 // occur if the first instruction must go in slot0.
1539 if (LoadJ && LoadI && HII->isPureSlot0(MI: J)) {
1540 FoundSequentialDependence = true;
1541 break;
1542 }
1543
1544 if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() &&
1545 ((LoadJ && StoreI && !NVStoreI) ||
1546 (StoreJ && LoadI && !NVStoreJ)) &&
1547 (J.getOpcode() != Hexagon::S2_allocframe &&
1548 I.getOpcode() != Hexagon::S2_allocframe) &&
1549 (J.getOpcode() != Hexagon::L2_deallocframe &&
1550 I.getOpcode() != Hexagon::L2_deallocframe) &&
1551 (!HII->isMemOp(MI: J) && !HII->isMemOp(MI: I)) && (!IsVecJ && !IsVecI))
1552 setmemShufDisabled(true);
1553 else
1554 if (StoreJ && LoadI && alias(MI1: J, MI2: I)) {
1555 FoundSequentialDependence = true;
1556 break;
1557 }
1558
1559 if (!StoreJ)
1560 if (!LoadJ || (!LoadI && !StoreI)) {
1561 // If J is neither load nor store, assume a dependency.
1562 // If J is a load, but I is neither, also assume a dependency.
1563 FoundSequentialDependence = true;
1564 break;
1565 }
1566 // Store followed by store: not OK on V2.
1567 // Store followed by load: not OK on all.
1568 // Load followed by store: OK on all.
1569 // Load followed by load: OK on all.
1570 continue;
1571 }
1572
1573 // Special case for ALLOCFRAME: even though there is dependency
1574 // between ALLOCFRAME and subsequent store, allow it to be packetized
1575 // in a same packet. This implies that the store is using the caller's
1576 // SP. Hence, offset needs to be updated accordingly.
1577 if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
1578 unsigned Opc = I.getOpcode();
1579 switch (Opc) {
1580 case Hexagon::S2_storerd_io:
1581 case Hexagon::S2_storeri_io:
1582 case Hexagon::S2_storerh_io:
1583 case Hexagon::S2_storerb_io:
1584 if (I.getOperand(i: 0).getReg() == HRI->getStackRegister()) {
1585 // Since this store is to be glued with allocframe in the same
1586 // packet, it will use SP of the previous stack frame, i.e.
1587 // caller's SP. Therefore, we need to recalculate offset
1588 // according to this change.
1589 GlueAllocframeStore = useCallersSP(MI&: I);
1590 if (GlueAllocframeStore)
1591 continue;
1592 }
1593 break;
1594 default:
1595 break;
1596 }
1597 }
1598
1599 // There are certain anti-dependencies that cannot be ignored.
1600 // Specifically:
1601 // J2_call ... implicit-def %r0 ; SUJ
1602 // R0 = ... ; SUI
1603 // Those cannot be packetized together, since the call will observe
1604 // the effect of the assignment to R0.
1605 if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) {
1606 // Check if I defines any volatile register. We should also check
1607 // registers that the call may read, but these happen to be a
1608 // subset of the volatile register set.
1609 for (const MachineOperand &Op : I.operands()) {
1610 if (Op.isReg() && Op.isDef()) {
1611 Register R = Op.getReg();
1612 if (!J.readsRegister(Reg: R, TRI: HRI) && !J.modifiesRegister(Reg: R, TRI: HRI))
1613 continue;
1614 } else if (!Op.isRegMask()) {
1615 // If I has a regmask assume dependency.
1616 continue;
1617 }
1618 FoundSequentialDependence = true;
1619 break;
1620 }
1621 }
1622
1623 // Skip over remaining anti-dependences. Two instructions that are
1624 // anti-dependent can share a packet, since in most such cases all
1625 // operands are read before any modifications take place.
1626 // The exceptions are branch and call instructions, since they are
1627 // executed after all other instructions have completed (at least
1628 // conceptually).
1629 if (DepType != SDep::Anti) {
1630 FoundSequentialDependence = true;
1631 break;
1632 }
1633 }
1634
1635 if (FoundSequentialDependence) {
1636 Dependence = true;
1637 return false;
1638 }
1639
1640 return true;
1641}
1642
1643bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
1644 assert(SUI->getInstr() && SUJ->getInstr());
1645 MachineInstr &I = *SUI->getInstr();
1646 MachineInstr &J = *SUJ->getInstr();
1647
1648 bool Coexist = !cannotCoexist(MI: I, MJ: J);
1649
1650 if (Coexist && !Dependence)
1651 return true;
1652
1653 // Check if the instruction was promoted to a dot-new. If so, demote it
1654 // back into a dot-old.
1655 if (PromotedToDotNew)
1656 demoteToDotOld(MI&: I);
1657
1658 cleanUpDotCur();
1659 // Check if the instruction (must be a store) was glued with an allocframe
1660 // instruction. If so, restore its offset to its original value, i.e. use
1661 // current SP instead of caller's SP.
1662 if (GlueAllocframeStore) {
1663 useCalleesSP(MI&: I);
1664 GlueAllocframeStore = false;
1665 }
1666
1667 if (ChangedOffset != INT64_MAX)
1668 undoChangedOffset(MI&: I);
1669
1670 if (GlueToNewValueJump) {
1671 // Putting I and J together would prevent the new-value jump from being
1672 // packetized with the producer. In that case I and J must be separated.
1673 GlueToNewValueJump = false;
1674 return false;
1675 }
1676
1677 if (!Coexist)
1678 return false;
1679
1680 if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) {
1681 FoundSequentialDependence = false;
1682 Dependence = false;
1683 return true;
1684 }
1685
1686 return false;
1687}
1688
1689
1690bool HexagonPacketizerList::foundLSInPacket() {
1691 bool FoundLoad = false;
1692 bool FoundStore = false;
1693
1694 for (auto *MJ : CurrentPacketMIs) {
1695 unsigned Opc = MJ->getOpcode();
1696 if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe)
1697 continue;
1698 if (HII->isMemOp(MI: *MJ))
1699 continue;
1700 if (MJ->mayLoad())
1701 FoundLoad = true;
1702 if (MJ->mayStore() && !HII->isNewValueStore(MI: *MJ))
1703 FoundStore = true;
1704 }
1705 return FoundLoad && FoundStore;
1706}
1707
1708
1709MachineBasicBlock::iterator
1710HexagonPacketizerList::addToPacket(MachineInstr &MI) {
1711 MachineBasicBlock::iterator MII = MI.getIterator();
1712 MachineBasicBlock *MBB = MI.getParent();
1713
1714 if (CurrentPacketMIs.empty()) {
1715 PacketStalls = false;
1716 PacketStallCycles = 0;
1717 }
1718 PacketStalls |= producesStall(MI);
1719 PacketStallCycles = std::max(a: PacketStallCycles, b: calcStall(MI));
1720
1721 if (MI.isImplicitDef()) {
1722 // Add to the packet to allow subsequent instructions to be checked
1723 // properly.
1724 CurrentPacketMIs.push_back(x: &MI);
1725 return MII;
1726 }
1727 assert(ResourceTracker->canReserveResources(MI));
1728
1729 bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
1730 bool Good = true;
1731
1732 if (GlueToNewValueJump) {
1733 MachineInstr &NvjMI = *++MII;
1734 // We need to put both instructions in the same packet: MI and NvjMI.
1735 // Either of them can require a constant extender. Try to add both to
1736 // the current packet, and if that fails, end the packet and start a
1737 // new one.
1738 ResourceTracker->reserveResources(MI);
1739 if (ExtMI)
1740 Good = tryAllocateResourcesForConstExt(Reserve: true);
1741
1742 bool ExtNvjMI = HII->isExtended(MI: NvjMI) || HII->isConstExtended(MI: NvjMI);
1743 if (Good) {
1744 if (ResourceTracker->canReserveResources(MI&: NvjMI))
1745 ResourceTracker->reserveResources(MI&: NvjMI);
1746 else
1747 Good = false;
1748 }
1749 if (Good && ExtNvjMI)
1750 Good = tryAllocateResourcesForConstExt(Reserve: true);
1751
1752 if (!Good) {
1753 endPacket(MBB, MI);
1754 assert(ResourceTracker->canReserveResources(MI));
1755 ResourceTracker->reserveResources(MI);
1756 if (ExtMI) {
1757 assert(canReserveResourcesForConstExt());
1758 tryAllocateResourcesForConstExt(Reserve: true);
1759 }
1760 assert(ResourceTracker->canReserveResources(NvjMI));
1761 ResourceTracker->reserveResources(MI&: NvjMI);
1762 if (ExtNvjMI) {
1763 assert(canReserveResourcesForConstExt());
1764 reserveResourcesForConstExt();
1765 }
1766 }
1767 CurrentPacketMIs.push_back(x: &MI);
1768 CurrentPacketMIs.push_back(x: &NvjMI);
1769 return MII;
1770 }
1771
1772 ResourceTracker->reserveResources(MI);
1773 if (ExtMI && !tryAllocateResourcesForConstExt(Reserve: true)) {
1774 endPacket(MBB, MI);
1775 if (PromotedToDotNew)
1776 demoteToDotOld(MI);
1777 if (GlueAllocframeStore) {
1778 useCalleesSP(MI);
1779 GlueAllocframeStore = false;
1780 }
1781 ResourceTracker->reserveResources(MI);
1782 reserveResourcesForConstExt();
1783 }
1784
1785 CurrentPacketMIs.push_back(x: &MI);
1786 return MII;
1787}
1788
1789void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
1790 MachineBasicBlock::iterator EndMI) {
1791 // Replace VLIWPacketizerList::endPacket(MBB, EndMI).
1792 LLVM_DEBUG({
1793 if (!CurrentPacketMIs.empty()) {
1794 dbgs() << "Finalizing packet:\n";
1795 unsigned Idx = 0;
1796 for (MachineInstr *MI : CurrentPacketMIs) {
1797 unsigned R = ResourceTracker->getUsedResources(Idx++);
1798 dbgs() << " * [res:0x" << utohexstr(R) << "] " << *MI;
1799 }
1800 }
1801 });
1802
1803 bool memShufDisabled = getmemShufDisabled();
1804 if (memShufDisabled && !foundLSInPacket()) {
1805 setmemShufDisabled(false);
1806 LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n");
1807 }
1808 memShufDisabled = getmemShufDisabled();
1809
1810 OldPacketMIs.clear();
1811 for (MachineInstr *MI : CurrentPacketMIs) {
1812 MachineBasicBlock::instr_iterator NextMI = std::next(x: MI->getIterator());
1813 for (auto &I : make_range(x: HII->expandVGatherPseudo(MI&: *MI), y: NextMI))
1814 OldPacketMIs.push_back(x: &I);
1815 }
1816 CurrentPacketMIs.clear();
1817
1818 if (OldPacketMIs.size() > 1) {
1819 MachineBasicBlock::instr_iterator FirstMI(OldPacketMIs.front());
1820 MachineBasicBlock::instr_iterator LastMI(EndMI.getInstrIterator());
1821 finalizeBundle(MBB&: *MBB, FirstMI, LastMI);
1822 auto BundleMII = std::prev(x: FirstMI);
1823 if (memShufDisabled)
1824 HII->setBundleNoShuf(BundleMII);
1825
1826 setmemShufDisabled(false);
1827 }
1828
1829 PacketHasDuplex = false;
1830 PacketHasSLOT0OnlyInsn = false;
1831 ResourceTracker->clearResources();
1832 LLVM_DEBUG(dbgs() << "End packet\n");
1833}
1834
1835bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
1836 if (Minimal)
1837 return false;
1838
1839 if (producesStall(MI))
1840 return false;
1841
1842 // If TinyCore with Duplexes is enabled, check if this MI can form a Duplex
1843 // with any other instruction in the existing packet.
1844 auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>();
1845 // Constraint 1: Only one duplex allowed per packet.
1846 // Constraint 2: Consider duplex checks only if there is atleast one
1847 // instruction in a packet.
1848 // Constraint 3: If one of the existing instructions in the packet has a
1849 // SLOT0 only instruction that can not be duplexed, do not attempt to form
1850 // duplexes. (TODO: This will invalidate the L4_return* instructions to form a
1851 // duplex)
1852 if (HST.isTinyCoreWithDuplex() && CurrentPacketMIs.size() > 0 &&
1853 !PacketHasDuplex) {
1854 // Check for SLOT0 only non-duplexable instruction in packet.
1855 for (auto &MJ : CurrentPacketMIs)
1856 PacketHasSLOT0OnlyInsn |= HII->isPureSlot0(MI: *MJ);
1857 // Get the Big Core Opcode (dup_*).
1858 int Opcode = HII->getDuplexOpcode(MI, ForBigCore: false);
1859 if (Opcode >= 0) {
1860 // We now have an instruction that can be duplexed.
1861 for (auto &MJ : CurrentPacketMIs) {
1862 if (HII->isDuplexPair(MIa: MI, MIb: *MJ) && !PacketHasSLOT0OnlyInsn) {
1863 PacketHasDuplex = true;
1864 return true;
1865 }
1866 }
1867 // If it can not be duplexed, check if there is a valid transition in DFA
1868 // with the original opcode.
1869 MachineInstr &MIRef = const_cast<MachineInstr &>(MI);
1870 MIRef.setDesc(HII->get(Opcode));
1871 return ResourceTracker->canReserveResources(MI&: MIRef);
1872 }
1873 }
1874
1875 return true;
1876}
1877
1878// V60 forward scheduling.
1879unsigned int HexagonPacketizerList::calcStall(const MachineInstr &I) {
1880 // Check whether the previous packet is in a different loop. If this is the
1881 // case, there is little point in trying to avoid a stall because that would
1882 // favor the rare case (loop entry) over the common case (loop iteration).
1883 //
1884 // TODO: We should really be able to check all the incoming edges if this is
1885 // the first packet in a basic block, so we can avoid stalls from the loop
1886 // backedge.
1887 if (!OldPacketMIs.empty()) {
1888 auto *OldBB = OldPacketMIs.front()->getParent();
1889 auto *ThisBB = I.getParent();
1890 if (MLI->getLoopFor(BB: OldBB) != MLI->getLoopFor(BB: ThisBB))
1891 return 0;
1892 }
1893
1894 SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)];
1895 if (!SUI)
1896 return 0;
1897
1898 // If the latency is 0 and there is a data dependence between this
1899 // instruction and any instruction in the current packet, we disregard any
1900 // potential stalls due to the instructions in the previous packet. Most of
1901 // the instruction pairs that can go together in the same packet have 0
1902 // latency between them. The exceptions are
1903 // 1. NewValueJumps as they're generated much later and the latencies can't
1904 // be changed at that point.
1905 // 2. .cur instructions, if its consumer has a 0 latency successor (such as
1906 // .new). In this case, the latency between .cur and the consumer stays
1907 // non-zero even though we can have both .cur and .new in the same packet.
1908 // Changing the latency to 0 is not an option as it causes software pipeliner
1909 // to not pipeline in some cases.
1910
1911 // For Example:
1912 // {
1913 // I1: v6.cur = vmem(r0++#1)
1914 // I2: v7 = valign(v6,v4,r2)
1915 // I3: vmem(r5++#1) = v7.new
1916 // }
1917 // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2.
1918
1919 for (auto *J : CurrentPacketMIs) {
1920 SUnit *SUJ = MIToSUnit[J];
1921 for (auto &Pred : SUI->Preds)
1922 if (Pred.getSUnit() == SUJ)
1923 if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) ||
1924 HII->isNewValueJump(MI: I) || HII->isToBeScheduledASAP(MI1: *J, MI2: I))
1925 return 0;
1926 }
1927
1928 // Check if the latency is greater than one between this instruction and any
1929 // instruction in the previous packet.
1930 for (auto *J : OldPacketMIs) {
1931 SUnit *SUJ = MIToSUnit[J];
1932 for (auto &Pred : SUI->Preds)
1933 if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
1934 return Pred.getLatency();
1935 }
1936
1937 return 0;
1938}
1939
1940bool HexagonPacketizerList::producesStall(const MachineInstr &I) {
1941 unsigned int Latency = calcStall(I);
1942 if (Latency == 0)
1943 return false;
1944 // Ignore stall unless it stalls more than previous instruction in packet
1945 if (PacketStalls)
1946 return Latency > PacketStallCycles;
1947 return true;
1948}
1949
1950//===----------------------------------------------------------------------===//
1951// Public Constructor Functions
1952//===----------------------------------------------------------------------===//
1953
1954FunctionPass *llvm::createHexagonPacketizer(bool Minimal) {
1955 return new HexagonPacketizer(Minimal);
1956}
1957