1//===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements a simple VLIW packetizer using DFA. The packetizer works on
10// machine basic blocks. For each instruction I in BB, the packetizer consults
11// the DFA to see if machine resources are available to execute I. If so, the
12// packetizer checks if I depends on any instruction J in the current packet.
13// If no dependency is found, I is added to current packet and machine resource
14// is marked as taken. If any dependency is found, a target API call is made to
15// prune the dependence.
16//
17//===----------------------------------------------------------------------===//
18
19#include "HexagonVLIWPacketizer.h"
20#include "Hexagon.h"
21#include "HexagonInstrInfo.h"
22#include "HexagonRegisterInfo.h"
23#include "HexagonSubtarget.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseSet.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/AliasAnalysis.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
31#include "llvm/CodeGen/MachineDominators.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineFunctionPass.h"
35#include "llvm/CodeGen/MachineInstr.h"
36#include "llvm/CodeGen/MachineInstrBundle.h"
37#include "llvm/CodeGen/MachineLoopInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/ScheduleDAG.h"
40#include "llvm/CodeGen/TargetRegisterInfo.h"
41#include "llvm/CodeGen/TargetSubtargetInfo.h"
42#include "llvm/IR/DebugLoc.h"
43#include "llvm/InitializePasses.h"
44#include "llvm/MC/MCInstrDesc.h"
45#include "llvm/Pass.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/raw_ostream.h"
50#include <cassert>
51#include <cstdint>
52#include <iterator>
53
54using namespace llvm;
55
56#define DEBUG_TYPE "packets"
57
58cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
59 cl::desc("Disable Hexagon packetizer pass"));
60
61static cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden,
62 cl::init(Val: true),
63 cl::desc("Allow slot1 store and slot0 load"));
64
65static cl::opt<bool> PacketizeVolatiles(
66 "hexagon-packetize-volatiles", cl::Hidden, cl::init(Val: true),
67 cl::desc("Allow non-solo packetization of volatile memory references"));
68
69static cl::opt<bool>
70 EnableGenAllInsnClass("enable-gen-insn", cl::Hidden,
71 cl::desc("Generate all instruction with TC"));
72
73static cl::opt<bool>
74 DisableVecDblNVStores("disable-vecdbl-nv-stores", cl::Hidden,
75 cl::desc("Disable vector double new-value-stores"));
76
77extern cl::opt<bool> ScheduleInlineAsm;
78
79namespace {
80
81 class HexagonPacketizer : public MachineFunctionPass {
82 public:
83 static char ID;
84
85 HexagonPacketizer(bool Min = false)
86 : MachineFunctionPass(ID), Minimal(Min) {}
87
88 void getAnalysisUsage(AnalysisUsage &AU) const override {
89 AU.setPreservesCFG();
90 AU.addRequired<AAResultsWrapperPass>();
91 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
92 AU.addRequired<MachineLoopInfoWrapperPass>();
93 MachineFunctionPass::getAnalysisUsage(AU);
94 }
95
96 StringRef getPassName() const override { return "Hexagon Packetizer"; }
97 bool runOnMachineFunction(MachineFunction &Fn) override;
98
99 MachineFunctionProperties getRequiredProperties() const override {
100 return MachineFunctionProperties().setNoVRegs();
101 }
102
103 private:
104 const HexagonInstrInfo *HII = nullptr;
105 const HexagonRegisterInfo *HRI = nullptr;
106 const bool Minimal = false;
107 };
108
109} // end anonymous namespace
110
111char HexagonPacketizer::ID = 0;
112
113INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
114 "Hexagon Packetizer", false, false)
115INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
116INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
117INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
118INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
119INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",
120 "Hexagon Packetizer", false, false)
121
122HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
123 MachineLoopInfo &MLI, AAResults *AA,
124 const MachineBranchProbabilityInfo *MBPI, bool Minimal)
125 : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI),
126 Minimal(Minimal) {
127 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
128 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
129
130 addMutation(Mutation: std::make_unique<HexagonSubtarget::UsrOverflowMutation>());
131 addMutation(Mutation: std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
132 addMutation(Mutation: std::make_unique<HexagonSubtarget::HMXActivateWeightMutation>());
133 addMutation(Mutation: std::make_unique<HexagonSubtarget::BankConflictMutation>());
134 addMutation(Mutation: std::make_unique<HexagonSubtarget::HLXPseudoMutation>());
135}
136
137// Check if FirstI modifies a register that SecondI reads.
138static bool hasWriteToReadDep(const MachineInstr &FirstI,
139 const MachineInstr &SecondI,
140 const TargetRegisterInfo *TRI) {
141 for (auto &MO : FirstI.operands()) {
142 if (!MO.isReg() || !MO.isDef())
143 continue;
144 Register R = MO.getReg();
145 if (SecondI.readsRegister(Reg: R, TRI))
146 return true;
147 }
148 return false;
149}
150
151
152static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI,
153 MachineBasicBlock::iterator BundleIt, bool Before) {
154 MachineBasicBlock::instr_iterator InsertPt;
155 if (Before)
156 InsertPt = BundleIt.getInstrIterator();
157 else
158 InsertPt = std::next(x: BundleIt).getInstrIterator();
159
160 MachineBasicBlock &B = *MI.getParent();
161 // The instruction should at least be bundled with the preceding instruction
162 // (there will always be one, i.e. BUNDLE, if nothing else).
163 assert(MI.isBundledWithPred());
164 if (MI.isBundledWithSucc()) {
165 MI.clearFlag(Flag: MachineInstr::BundledSucc);
166 MI.clearFlag(Flag: MachineInstr::BundledPred);
167 } else {
168 // If it's not bundled with the successor (i.e. it is the last one
169 // in the bundle), then we can simply unbundle it from the predecessor,
170 // which will take care of updating the predecessor's flag.
171 MI.unbundleFromPred();
172 }
173 B.splice(Where: InsertPt, Other: &B, From: MI.getIterator());
174
175 // Get the size of the bundle without asserting.
176 MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
177 MachineBasicBlock::const_instr_iterator E = B.instr_end();
178 unsigned Size = 0;
179 for (++I; I != E && I->isBundledWithPred(); ++I)
180 ++Size;
181
182 // If there are still two or more instructions, then there is nothing
183 // else to be done.
184 if (Size > 1)
185 return BundleIt;
186
187 // Otherwise, extract the single instruction out and delete the bundle.
188 MachineBasicBlock::iterator NextIt = std::next(x: BundleIt);
189 MachineInstr &SingleI = *BundleIt->getNextNode();
190 SingleI.unbundleFromPred();
191 assert(!SingleI.isBundledWithSucc());
192 BundleIt->eraseFromParent();
193 return NextIt;
194}
195
196bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
197 // FIXME: This pass causes verification failures.
198 MF.getProperties().setFailsVerification();
199
200 auto &HST = MF.getSubtarget<HexagonSubtarget>();
201 HII = HST.getInstrInfo();
202 HRI = HST.getRegisterInfo();
203 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
204 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
205 auto *MBPI =
206 &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
207
208 if (EnableGenAllInsnClass)
209 HII->genAllInsnTimingClasses(MF);
210
211 // Instantiate the packetizer.
212 bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() ||
213 skipFunction(F: MF.getFunction());
214 HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI, MinOnly);
215
216 // DFA state table should not be empty.
217 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
218
219 // Loop over all basic blocks and remove KILL pseudo-instructions
220 // These instructions confuse the dependence analysis. Consider:
221 // D0 = ... (Insn 0)
222 // R0 = KILL R0, D0 (Insn 1)
223 // R0 = ... (Insn 2)
224 // Here, Insn 1 will result in the dependence graph not emitting an output
225 // dependence between Insn 0 and Insn 2. This can lead to incorrect
226 // packetization
227 for (MachineBasicBlock &MB : MF) {
228 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MB))
229 if (MI.isKill())
230 MB.erase(I: &MI);
231 }
232
233 // TinyCore with Duplexes: Translate to big-instructions.
234 if (HST.isTinyCoreWithDuplex())
235 HII->translateInstrsForDup(MF, ToBigInstrs: true);
236
237 // Loop over all of the basic blocks.
238 for (auto &MB : MF) {
239 auto Begin = MB.begin(), End = MB.end();
240 while (Begin != End) {
241 // Find the first non-boundary starting from the end of the last
242 // scheduling region.
243 MachineBasicBlock::iterator RB = Begin;
244 while (RB != End && HII->isSchedulingBoundary(MI: *RB, MBB: &MB, MF))
245 ++RB;
246 // Find the first boundary starting from the beginning of the new
247 // region.
248 MachineBasicBlock::iterator RE = RB;
249 while (RE != End && !HII->isSchedulingBoundary(MI: *RE, MBB: &MB, MF))
250 ++RE;
251 // Add the scheduling boundary if it's not block end.
252 if (RE != End)
253 ++RE;
254 // If RB == End, then RE == End.
255 if (RB != End)
256 Packetizer.PacketizeMIs(MBB: &MB, BeginItr: RB, EndItr: RE);
257
258 Begin = RE;
259 }
260 }
261
262 // TinyCore with Duplexes: Translate to tiny-instructions.
263 if (HST.isTinyCoreWithDuplex())
264 HII->translateInstrsForDup(MF, ToBigInstrs: false);
265
266 Packetizer.unpacketizeSoloInstrs(MF);
267 return true;
268}
269
270// Reserve resources for a constant extender. Trigger an assertion if the
271// reservation fails.
272void HexagonPacketizerList::reserveResourcesForConstExt() {
273 if (!tryAllocateResourcesForConstExt(Reserve: true))
274 llvm_unreachable("Resources not available");
275}
276
277bool HexagonPacketizerList::canReserveResourcesForConstExt() {
278 return tryAllocateResourcesForConstExt(Reserve: false);
279}
280
281// Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
282// return true, otherwise, return false.
283bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
284 auto *ExtMI = MF.CreateMachineInstr(MCID: HII->get(Opcode: Hexagon::A4_ext), DL: DebugLoc());
285 bool Avail = ResourceTracker->canReserveResources(MI&: *ExtMI);
286 if (Reserve && Avail)
287 ResourceTracker->reserveResources(MI&: *ExtMI);
288 MF.deleteMachineInstr(MI: ExtMI);
289 return Avail;
290}
291
292bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI,
293 SDep::Kind DepType, unsigned DepReg) {
294 // Check for LR dependence.
295 if (DepReg == HRI->getRARegister())
296 return true;
297
298 if (HII->isDeallocRet(MI))
299 if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
300 return true;
301
302 // Call-like instructions can be packetized with preceding instructions
303 // that define registers implicitly used or modified by the call. Explicit
304 // uses are still prohibited, as in the case of indirect calls:
305 // r0 = ...
306 // J2_jumpr r0
307 if (DepType == SDep::Data) {
308 for (const MachineOperand &MO : MI.operands())
309 if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())
310 return true;
311 }
312
313 return false;
314}
315
316static bool isRegDependence(const SDep::Kind DepType) {
317 return DepType == SDep::Data || DepType == SDep::Anti ||
318 DepType == SDep::Output;
319}
320
321static bool isDirectJump(const MachineInstr &MI) {
322 return MI.getOpcode() == Hexagon::J2_jump;
323}
324
325static bool isSchedBarrier(const MachineInstr &MI) {
326 switch (MI.getOpcode()) {
327 case Hexagon::Y2_barrier:
328 return true;
329 }
330 return false;
331}
332
333static bool isControlFlow(const MachineInstr &MI) {
334 return MI.getDesc().isTerminator() || MI.getDesc().isCall();
335}
336
337/// Returns true if the instruction modifies a callee-saved register.
338static bool doesModifyCalleeSavedReg(const MachineInstr &MI,
339 const TargetRegisterInfo *TRI) {
340 const MachineFunction &MF = *MI.getParent()->getParent();
341 for (auto *CSR = TRI->getCalleeSavedRegs(MF: &MF); CSR && *CSR; ++CSR)
342 if (MI.modifiesRegister(Reg: *CSR, TRI))
343 return true;
344 return false;
345}
346
347// Returns true if an instruction can be promoted to .new predicate or
348// new-value store.
349bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI,
350 const TargetRegisterClass *NewRC) {
351 // Vector stores can be predicated, and can be new-value stores, but
352 // they cannot be predicated on a .new predicate value.
353 if (NewRC == &Hexagon::PredRegsRegClass) {
354 if (HII->isHVXVec(MI) && MI.mayStore())
355 return false;
356 return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, MBPI: nullptr) > 0;
357 }
358 // If the class is not PredRegs, it could only apply to new-value stores.
359 return HII->mayBeNewStore(MI);
360}
361
362// Promote an instructiont to its .cur form.
363// At this time, we have already made a call to canPromoteToDotCur and made
364// sure that it can *indeed* be promoted.
365bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI,
366 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
367 const TargetRegisterClass* RC) {
368 assert(DepType == SDep::Data);
369 int CurOpcode = HII->getDotCurOp(MI);
370 MI.setDesc(HII->get(Opcode: CurOpcode));
371 return true;
372}
373
374void HexagonPacketizerList::cleanUpDotCur() {
375 MachineInstr *MI = nullptr;
376 for (auto *BI : CurrentPacketMIs) {
377 LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
378 if (HII->isDotCurInst(MI: *BI)) {
379 MI = BI;
380 continue;
381 }
382 if (MI) {
383 for (auto &MO : BI->operands())
384 if (MO.isReg() && MO.getReg() == MI->getOperand(i: 0).getReg())
385 return;
386 }
387 }
388 if (!MI)
389 return;
390 // We did not find a use of the CUR, so de-cur it.
391 MI->setDesc(HII->get(Opcode: HII->getNonDotCurOp(MI: *MI)));
392 LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
393}
394
395// Check to see if an instruction can be dot cur.
396bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
397 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
398 const TargetRegisterClass *RC) {
399 if (!HII->isHVXVec(MI))
400 return false;
401 if (!HII->isHVXVec(MI: *MII))
402 return false;
403
404 // Already a dot new instruction.
405 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
406 return false;
407
408 if (!HII->mayBeCurLoad(MI))
409 return false;
410
411 // The "cur value" cannot come from inline asm.
412 if (PacketSU->getInstr()->isInlineAsm())
413 return false;
414
415 // Make sure candidate instruction uses cur.
416 LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump();
417 dbgs() << "in packet\n";);
418 MachineInstr &MJ = *MII;
419 LLVM_DEBUG({
420 dbgs() << "Checking CUR against ";
421 MJ.dump();
422 });
423 Register DestReg = MI.getOperand(i: 0).getReg();
424 bool FoundMatch = false;
425 for (auto &MO : MJ.operands())
426 if (MO.isReg() && MO.getReg() == DestReg)
427 FoundMatch = true;
428 if (!FoundMatch)
429 return false;
430
431 // Check for existing uses of a vector register within the packet which
432 // would be affected by converting a vector load into .cur format.
433 for (auto *BI : CurrentPacketMIs) {
434 LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););
435 if (BI->readsRegister(Reg: DepReg, TRI: MF.getSubtarget().getRegisterInfo()))
436 return false;
437 }
438
439 LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
440 // We can convert the opcode into a .cur.
441 return true;
442}
443
444// Promote an instruction to its .new form. At this time, we have already
445// made a call to canPromoteToDotNew and made sure that it can *indeed* be
446// promoted.
447bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI,
448 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
449 const TargetRegisterClass* RC) {
450 assert(DepType == SDep::Data);
451 int NewOpcode;
452 if (RC == &Hexagon::PredRegsRegClass)
453 NewOpcode = HII->getDotNewPredOp(MI, MBPI);
454 else
455 NewOpcode = HII->getDotNewOp(MI);
456 MI.setDesc(HII->get(Opcode: NewOpcode));
457 return true;
458}
459
460bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) {
461 int NewOpcode = HII->getDotOldOp(MI);
462 MI.setDesc(HII->get(Opcode: NewOpcode));
463 return true;
464}
465
466bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
467 unsigned Opc = MI.getOpcode();
468 switch (Opc) {
469 case Hexagon::S2_storerd_io:
470 case Hexagon::S2_storeri_io:
471 case Hexagon::S2_storerh_io:
472 case Hexagon::S2_storerb_io:
473 break;
474 default:
475 llvm_unreachable("Unexpected instruction");
476 }
477 unsigned FrameSize = MF.getFrameInfo().getStackSize();
478 MachineOperand &Off = MI.getOperand(i: 1);
479 int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
480 if (HII->isValidOffset(Opcode: Opc, Offset: NewOff)) {
481 Off.setImm(NewOff);
482 return true;
483 }
484 return false;
485}
486
487void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) {
488 unsigned Opc = MI.getOpcode();
489 switch (Opc) {
490 case Hexagon::S2_storerd_io:
491 case Hexagon::S2_storeri_io:
492 case Hexagon::S2_storerh_io:
493 case Hexagon::S2_storerb_io:
494 break;
495 default:
496 llvm_unreachable("Unexpected instruction");
497 }
498 unsigned FrameSize = MF.getFrameInfo().getStackSize();
499 MachineOperand &Off = MI.getOperand(i: 1);
500 Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
501}
502
503/// Return true if we can update the offset in MI so that MI and MJ
504/// can be packetized together.
505bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) {
506 assert(SUI->getInstr() && SUJ->getInstr());
507 MachineInstr &MI = *SUI->getInstr();
508 MachineInstr &MJ = *SUJ->getInstr();
509
510 unsigned BPI, OPI;
511 if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BPI, OffsetPos&: OPI))
512 return false;
513 unsigned BPJ, OPJ;
514 if (!HII->getBaseAndOffsetPosition(MI: MJ, BasePos&: BPJ, OffsetPos&: OPJ))
515 return false;
516 Register Reg = MI.getOperand(i: BPI).getReg();
517 if (Reg != MJ.getOperand(i: BPJ).getReg())
518 return false;
519 // Make sure that the dependences do not restrict adding MI to the packet.
520 // That is, ignore anti dependences, and make sure the only data dependence
521 // involves the specific register.
522 for (const auto &PI : SUI->Preds)
523 if (PI.getKind() != SDep::Anti &&
524 (PI.getKind() != SDep::Data || PI.getReg() != Reg))
525 return false;
526 int Incr;
527 if (!HII->getIncrementValue(MI: MJ, Value&: Incr))
528 return false;
529
530 int64_t Offset = MI.getOperand(i: OPI).getImm();
531 if (!HII->isValidOffset(Opcode: MI.getOpcode(), Offset: Offset + Incr))
532 return false;
533
534 MI.getOperand(i: OPI).setImm(Offset + Incr);
535 ChangedOffset = Offset;
536 return true;
537}
538
539/// Undo the changed offset. This is needed if the instruction cannot be
540/// added to the current packet due to a different instruction.
541void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) {
542 unsigned BP, OP;
543 if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BP, OffsetPos&: OP))
544 llvm_unreachable("Unable to find base and offset operands.");
545 MI.getOperand(i: OP).setImm(ChangedOffset);
546}
547
548enum PredicateKind {
549 PK_False,
550 PK_True,
551 PK_Unknown
552};
553
554/// Returns true if an instruction is predicated on p0 and false if it's
555/// predicated on !p0.
556static PredicateKind getPredicateSense(const MachineInstr &MI,
557 const HexagonInstrInfo *HII) {
558 if (!HII->isPredicated(MI))
559 return PK_Unknown;
560 if (HII->isPredicatedTrue(MI))
561 return PK_True;
562 return PK_False;
563}
564
565static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI,
566 const HexagonInstrInfo *HII) {
567 assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
568#ifndef NDEBUG
569 // Post Increment means duplicates. Use dense map to find duplicates in the
570 // list. Caution: Densemap initializes with the minimum of 64 buckets,
571 // whereas there are at most 5 operands in the post increment.
572 DenseSet<unsigned> DefRegsSet;
573 for (auto &MO : MI.operands())
574 if (MO.isReg() && MO.isDef())
575 DefRegsSet.insert(MO.getReg());
576
577 for (auto &MO : MI.operands())
578 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
579 return MO;
580#else
581 if (MI.mayLoad()) {
582 const MachineOperand &Op1 = MI.getOperand(i: 1);
583 // The 2nd operand is always the post increment operand in load.
584 assert(Op1.isReg() && "Post increment operand has be to a register.");
585 return Op1;
586 }
587 if (MI.getDesc().mayStore()) {
588 const MachineOperand &Op0 = MI.getOperand(i: 0);
589 // The 1st operand is always the post increment operand in store.
590 assert(Op0.isReg() && "Post increment operand has be to a register.");
591 return Op0;
592 }
593#endif
594 // we should never come here.
595 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
596}
597
598// Get the value being stored.
599static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) {
600 // value being stored is always the last operand.
601 return MI.getOperand(i: MI.getNumOperands()-1);
602}
603
604static bool isLoadAbsSet(const MachineInstr &MI) {
605 unsigned Opc = MI.getOpcode();
606 switch (Opc) {
607 case Hexagon::L4_loadrd_ap:
608 case Hexagon::L4_loadrb_ap:
609 case Hexagon::L4_loadrh_ap:
610 case Hexagon::L4_loadrub_ap:
611 case Hexagon::L4_loadruh_ap:
612 case Hexagon::L4_loadri_ap:
613 return true;
614 }
615 return false;
616}
617
618static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) {
619 assert(isLoadAbsSet(MI));
620 return MI.getOperand(i: 1);
621}
622
623// Can be new value store?
624// Following restrictions are to be respected in convert a store into
625// a new value store.
626// 1. If an instruction uses auto-increment, its address register cannot
627// be a new-value register. Arch Spec 5.4.2.1
628// 2. If an instruction uses absolute-set addressing mode, its address
629// register cannot be a new-value register. Arch Spec 5.4.2.1.
630// 3. If an instruction produces a 64-bit result, its registers cannot be used
631// as new-value registers. Arch Spec 5.4.2.2.
632// 4. If the instruction that sets the new-value register is conditional, then
633// the instruction that uses the new-value register must also be conditional,
634// and both must always have their predicates evaluate identically.
635// Arch Spec 5.4.2.3.
636// 5. There is an implied restriction that a packet cannot have another store,
637// if there is a new value store in the packet. Corollary: if there is
638// already a store in a packet, there can not be a new value store.
639// Arch Spec: 3.4.4.2
640bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
641 const MachineInstr &PacketMI, unsigned DepReg) {
642 // Make sure we are looking at the store, that can be promoted.
643 if (!HII->mayBeNewStore(MI))
644 return false;
645
646 // Make sure there is dependency and can be new value'd.
647 const MachineOperand &Val = getStoreValueOperand(MI);
648 if (Val.isReg() && Val.getReg() != DepReg)
649 return false;
650
651 const MCInstrDesc& MCID = PacketMI.getDesc();
652
653 // First operand is always the result.
654 const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, OpNum: 0);
655 // Double regs can not feed into new value store: PRM section: 5.4.2.2.
656 if (PacketRC == &Hexagon::DoubleRegsRegClass)
657 return false;
658
659 // New-value stores are of class NV (slot 0), dual stores require class ST
660 // in slot 0 (PRM 5.5).
661 for (auto *I : CurrentPacketMIs) {
662 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
663 if (PacketSU->getInstr()->mayStore())
664 return false;
665 }
666
667 // Make sure it's NOT the post increment register that we are going to
668 // new value.
669 if (HII->isPostIncrement(MI) &&
670 getPostIncrementOperand(MI, HII).getReg() == DepReg) {
671 return false;
672 }
673
674 if (HII->isPostIncrement(MI: PacketMI) && PacketMI.mayLoad() &&
675 getPostIncrementOperand(MI: PacketMI, HII).getReg() == DepReg) {
676 // If source is post_inc, or absolute-set addressing, it can not feed
677 // into new value store
678 // r3 = memw(r2++#4)
679 // memw(r30 + #-1404) = r2.new -> can not be new value store
680 // arch spec section: 5.4.2.1.
681 return false;
682 }
683
684 if (isLoadAbsSet(MI: PacketMI) && getAbsSetOperand(MI: PacketMI).getReg() == DepReg)
685 return false;
686
687 // If the source that feeds the store is predicated, new value store must
688 // also be predicated.
689 if (HII->isPredicated(MI: PacketMI)) {
690 if (!HII->isPredicated(MI))
691 return false;
692
693 // Check to make sure that they both will have their predicates
694 // evaluate identically.
695 unsigned predRegNumSrc = 0;
696 unsigned predRegNumDst = 0;
697 const TargetRegisterClass* predRegClass = nullptr;
698
699 // Get predicate register used in the source instruction.
700 for (auto &MO : PacketMI.operands()) {
701 if (!MO.isReg())
702 continue;
703 predRegNumSrc = MO.getReg();
704 predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumSrc);
705 if (predRegClass == &Hexagon::PredRegsRegClass)
706 break;
707 }
708 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
709 "predicate register not found in a predicated PacketMI instruction");
710
711 // Get predicate register used in new-value store instruction.
712 for (auto &MO : MI.operands()) {
713 if (!MO.isReg())
714 continue;
715 predRegNumDst = MO.getReg();
716 predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumDst);
717 if (predRegClass == &Hexagon::PredRegsRegClass)
718 break;
719 }
720 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
721 "predicate register not found in a predicated MI instruction");
722
723 // New-value register producer and user (store) need to satisfy these
724 // constraints:
725 // 1) Both instructions should be predicated on the same register.
726 // 2) If producer of the new-value register is .new predicated then store
727 // should also be .new predicated and if producer is not .new predicated
728 // then store should not be .new predicated.
729 // 3) Both new-value register producer and user should have same predicate
730 // sense, i.e, either both should be negated or both should be non-negated.
731 if (predRegNumDst != predRegNumSrc ||
732 HII->isDotNewInst(MI: PacketMI) != HII->isDotNewInst(MI) ||
733 getPredicateSense(MI, HII) != getPredicateSense(MI: PacketMI, HII))
734 return false;
735 }
736
737 // Make sure that other than the new-value register no other store instruction
738 // register has been modified in the same packet. Predicate registers can be
739 // modified by they should not be modified between the producer and the store
740 // instruction as it will make them both conditional on different values.
741 // We already know this to be true for all the instructions before and
742 // including PacketMI. However, we need to perform the check for the
743 // remaining instructions in the packet.
744
745 unsigned StartCheck = 0;
746
747 for (auto *I : CurrentPacketMIs) {
748 SUnit *TempSU = MIToSUnit.find(x: I)->second;
749 MachineInstr &TempMI = *TempSU->getInstr();
750
751 // Following condition is true for all the instructions until PacketMI is
752 // reached (StartCheck is set to 0 before the for loop).
753 // StartCheck flag is 1 for all the instructions after PacketMI.
754 if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
755 continue; // encountering PacketMI.
756
757 StartCheck = 1;
758 if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
759 continue;
760
761 for (auto &MO : MI.operands())
762 if (MO.isReg() && TempSU->getInstr()->modifiesRegister(Reg: MO.getReg(), TRI: HRI))
763 return false;
764 }
765
766 // Make sure that for non-POST_INC stores:
767 // 1. The only use of reg is DepReg and no other registers.
768 // This handles base+index registers.
769 // The following store can not be dot new.
770 // Eg. r0 = add(r0, #3)
771 // memw(r1+r0<<#2) = r0
772 if (!HII->isPostIncrement(MI)) {
773 for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
774 const MachineOperand &MO = MI.getOperand(i: opNum);
775 if (MO.isReg() && MO.getReg() == DepReg)
776 return false;
777 }
778 }
779
780 // If data definition is because of implicit definition of the register,
781 // do not newify the store. Eg.
782 // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12
783 // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343]
784 for (auto &MO : PacketMI.operands()) {
785 if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg))
786 return false;
787 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
788 continue;
789 Register R = MO.getReg();
790 if (R == DepReg || HRI->isSuperRegister(RegA: DepReg, RegB: R))
791 return false;
792 }
793
794 // Handle imp-use of super reg case. There is a target independent side
795 // change that should prevent this situation but I am handling it for
796 // just-in-case. For example, we cannot newify R2 in the following case:
797 // %r3 = A2_tfrsi 0;
798 // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1;
799 for (auto &MO : MI.operands()) {
800 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
801 return false;
802 }
803
804 // Can be dot new store.
805 return true;
806}
807
808// Can this MI to promoted to either new value store or new value jump.
809bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI,
810 const SUnit *PacketSU, unsigned DepReg,
811 MachineBasicBlock::iterator &MII) {
812 if (!HII->mayBeNewStore(MI))
813 return false;
814
815 // Check to see the store can be new value'ed.
816 MachineInstr &PacketMI = *PacketSU->getInstr();
817 if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
818 return true;
819
820 // Check to see the compare/jump can be new value'ed.
821 // This is done as a pass on its own. Don't need to check it here.
822 return false;
823}
824
825static bool isImplicitDependency(const MachineInstr &I, bool CheckDef,
826 unsigned DepReg) {
827 for (auto &MO : I.operands()) {
828 if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg))
829 return true;
830 if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())
831 continue;
832 if (CheckDef == MO.isDef())
833 return true;
834 }
835 return false;
836}
837
838// Check to see if an instruction can be dot new.
839bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
840 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
841 const TargetRegisterClass* RC) {
842 // Already a dot new instruction.
843 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
844 return false;
845
846 if (!isNewifiable(MI, NewRC: RC))
847 return false;
848
849 const MachineInstr &PI = *PacketSU->getInstr();
850
851 // The "new value" cannot come from inline asm.
852 if (PI.isInlineAsm())
853 return false;
854
855 // IMPLICIT_DEFs and FREEZEs won't materialize as real instructions, so .new
856 // makes no sense.
857 if (PI.isImplicitDef() || PI.getOpcode() == TargetOpcode::FREEZE)
858 return false;
859
860 // If dependency is through an implicitly defined register, we should not
861 // newify the use.
862 if (isImplicitDependency(I: PI, CheckDef: true, DepReg) ||
863 isImplicitDependency(I: MI, CheckDef: false, DepReg))
864 return false;
865
866 const MCInstrDesc& MCID = PI.getDesc();
867 const TargetRegisterClass *VecRC = HII->getRegClass(MCID, OpNum: 0);
868 if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass)
869 return false;
870
871 // predicate .new
872 if (RC == &Hexagon::PredRegsRegClass)
873 return HII->predCanBeUsedAsDotNew(MI: PI, PredReg: DepReg);
874
875 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
876 return false;
877
878 // Create a dot new machine instruction to see if resources can be
879 // allocated. If not, bail out now.
880 int NewOpcode = (RC != &Hexagon::PredRegsRegClass) ? HII->getDotNewOp(MI) :
881 HII->getDotNewPredOp(MI, MBPI);
882 const MCInstrDesc &D = HII->get(Opcode: NewOpcode);
883 MachineInstr *NewMI = MF.CreateMachineInstr(MCID: D, DL: DebugLoc());
884 bool ResourcesAvailable = ResourceTracker->canReserveResources(MI&: *NewMI);
885 MF.deleteMachineInstr(MI: NewMI);
886 if (!ResourcesAvailable)
887 return false;
888
889 // New Value Store only. New Value Jump generated as a separate pass.
890 if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
891 return false;
892
893 return true;
894}
895
896// Go through the packet instructions and search for an anti dependency between
897// them and DepReg from MI. Consider this case:
898// Trying to add
899// a) %r1 = TFRI_cdNotPt %p3, 2
900// to this packet:
901// {
902// b) %p0 = C2_or killed %p3, killed %p0
903// c) %p3 = C2_tfrrp %r23
904// d) %r1 = C2_cmovenewit %p3, 4
905// }
906// The P3 from a) and d) will be complements after
907// a)'s P3 is converted to .new form
908// Anti-dep between c) and b) is irrelevant for this case
909bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI,
910 unsigned DepReg) {
911 SUnit *PacketSUDep = MIToSUnit.find(x: &MI)->second;
912
913 for (auto *I : CurrentPacketMIs) {
914 // We only care for dependencies to predicated instructions
915 if (!HII->isPredicated(MI: *I))
916 continue;
917
918 // Scheduling Unit for current insn in the packet
919 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
920
921 // Look at dependencies between current members of the packet and
922 // predicate defining instruction MI. Make sure that dependency is
923 // on the exact register we care about.
924 if (PacketSU->isSucc(N: PacketSUDep)) {
925 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
926 auto &Dep = PacketSU->Succs[i];
927 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
928 Dep.getReg() == DepReg)
929 return true;
930 }
931 }
932 }
933
934 return false;
935}
936
937/// Gets the predicate register of a predicated instruction.
938static unsigned getPredicatedRegister(MachineInstr &MI,
939 const HexagonInstrInfo *QII) {
940 /// We use the following rule: The first predicate register that is a use is
941 /// the predicate register of a predicated instruction.
942 assert(QII->isPredicated(MI) && "Must be predicated instruction");
943
944 for (auto &Op : MI.operands()) {
945 if (Op.isReg() && Op.getReg() && Op.isUse() &&
946 Hexagon::PredRegsRegClass.contains(Reg: Op.getReg()))
947 return Op.getReg();
948 }
949
950 llvm_unreachable("Unknown instruction operand layout");
951 return 0;
952}
953
954// Given two predicated instructions, this function detects whether
955// the predicates are complements.
956bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
957 MachineInstr &MI2) {
958 // If we don't know the predicate sense of the instructions bail out early, we
959 // need it later.
960 if (getPredicateSense(MI: MI1, HII) == PK_Unknown ||
961 getPredicateSense(MI: MI2, HII) == PK_Unknown)
962 return false;
963
964 // Scheduling unit for candidate.
965 SUnit *SU = MIToSUnit[&MI1];
966
967 // One corner case deals with the following scenario:
968 // Trying to add
969 // a) %r24 = A2_tfrt %p0, %r25
970 // to this packet:
971 // {
972 // b) %r25 = A2_tfrf %p0, %r24
973 // c) %p0 = C2_cmpeqi %r26, 1
974 // }
975 //
976 // On general check a) and b) are complements, but presence of c) will
977 // convert a) to .new form, and then it is not a complement.
978 // We attempt to detect it by analyzing existing dependencies in the packet.
979
980 // Analyze relationships between all existing members of the packet.
981 // Look for Anti dependency on the same predicate reg as used in the
982 // candidate.
983 for (auto *I : CurrentPacketMIs) {
984 // Scheduling Unit for current insn in the packet.
985 SUnit *PacketSU = MIToSUnit.find(x: I)->second;
986
987 // If this instruction in the packet is succeeded by the candidate...
988 if (PacketSU->isSucc(N: SU)) {
989 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
990 auto Dep = PacketSU->Succs[i];
991 // The corner case exist when there is true data dependency between
992 // candidate and one of current packet members, this dep is on
993 // predicate reg, and there already exist anti dep on the same pred in
994 // the packet.
995 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
996 Hexagon::PredRegsRegClass.contains(Reg: Dep.getReg())) {
997 // Here I know that I is predicate setting instruction with true
998 // data dep to candidate on the register we care about - c) in the
999 // above example. Now I need to see if there is an anti dependency
1000 // from c) to any other instruction in the same packet on the pred
1001 // reg of interest.
1002 if (restrictingDepExistInPacket(MI&: *I, DepReg: Dep.getReg()))
1003 return false;
1004 }
1005 }
1006 }
1007 }
1008
1009 // If the above case does not apply, check regular complement condition.
1010 // Check that the predicate register is the same and that the predicate
1011 // sense is different We also need to differentiate .old vs. .new: !p0
1012 // is not complementary to p0.new.
1013 unsigned PReg1 = getPredicatedRegister(MI&: MI1, QII: HII);
1014 unsigned PReg2 = getPredicatedRegister(MI&: MI2, QII: HII);
1015 return PReg1 == PReg2 &&
1016 Hexagon::PredRegsRegClass.contains(Reg: PReg1) &&
1017 Hexagon::PredRegsRegClass.contains(Reg: PReg2) &&
1018 getPredicateSense(MI: MI1, HII) != getPredicateSense(MI: MI2, HII) &&
1019 HII->isDotNewInst(MI: MI1) == HII->isDotNewInst(MI: MI2);
1020}
1021
1022// Initialize packetizer flags.
1023void HexagonPacketizerList::initPacketizerState() {
1024 Dependence = false;
1025 PromotedToDotNew = false;
1026 GlueToNewValueJump = false;
1027 GlueAllocframeStore = false;
1028 FoundSequentialDependence = false;
1029 ChangedOffset = INT64_MAX;
1030}
1031
1032// Ignore bundling of pseudo instructions.
1033bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
1034 const MachineBasicBlock *) {
1035 if (MI.isDebugInstr())
1036 return true;
1037
1038 if (MI.isCFIInstruction())
1039 return false;
1040
1041 // We must print out inline assembly.
1042 if (MI.isInlineAsm())
1043 return false;
1044
1045 if (MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::FREEZE)
1046 return false;
1047
1048 // We check if MI has any functional units mapped to it. If it doesn't,
1049 // we ignore the instruction.
1050 const MCInstrDesc& TID = MI.getDesc();
1051 auto *IS = ResourceTracker->getInstrItins()->beginStage(ItinClassIndx: TID.getSchedClass());
1052 return !IS->getUnits();
1053}
1054
1055bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
1056 // Ensure any bundles created by gather packetize remain separate.
1057 if (MI.isBundle())
1058 return true;
1059
1060 if (MI.isEHLabel() || MI.isCFIInstruction())
1061 return true;
1062
1063 // Consider inline asm to not be a solo instruction by default.
1064 // Inline asm will be put in a packet temporarily, but then it will be
1065 // removed, and placed outside of the packet (before or after, depending
1066 // on dependencies). This is to reduce the impact of inline asm as a
1067 // "packet splitting" instruction.
1068 if (MI.isInlineAsm() && !ScheduleInlineAsm)
1069 return true;
1070
1071 if (isSchedBarrier(MI))
1072 return true;
1073
1074 if (HII->isSolo(MI))
1075 return true;
1076
1077 if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER ||
1078 MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT ||
1079 MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL ||
1080 MI.getOpcode() == Hexagon::PATCHABLE_EVENT_CALL ||
1081 MI.getOpcode() == Hexagon::PATCHABLE_TYPED_EVENT_CALL)
1082 return true;
1083
1084 if (MI.getOpcode() == Hexagon::A2_nop)
1085 return true;
1086
1087 return false;
1088}
1089
1090// Quick check if instructions MI and MJ cannot coexist in the same packet.
1091// Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
1092// but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1093// For full test call this function twice:
1094// cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
1095// Doing the test only one way saves the amount of code in this function,
1096// since every test would need to be repeated with the MI and MJ reversed.
1097static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
1098 const HexagonInstrInfo &HII) {
1099 const MachineFunction *MF = MI.getParent()->getParent();
1100 if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() &&
1101 HII.isHVXMemWithAIndirect(I: MI, J: MJ))
1102 return true;
1103
1104 // Don't allow a store and an instruction that must be in slot0 and
1105 // doesn't allow a slot1 instruction.
1106 if (MI.mayStore() && HII.isRestrictNoSlot1Store(MI: MJ) && HII.isPureSlot0(MI: MJ))
1107 return true;
1108
1109 // An inline asm cannot be together with a branch, because we may not be
1110 // able to remove the asm out after packetizing (i.e. if the asm must be
1111 // moved past the bundle). Similarly, two asms cannot be together to avoid
1112 // complications when determining their relative order outside of a bundle.
1113 if (MI.isInlineAsm())
1114 return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
1115 MJ.isCall() || MJ.isTerminator();
1116
1117 // New-value stores cannot coexist with any other stores.
1118 if (HII.isNewValueStore(MI) && MJ.mayStore())
1119 return true;
1120
1121 // An HLX instruction with a 64-bit scalar source cannot be paired with
1122 // another HLX instruction using a scalar or immediate source. Loads and
1123 // stores are exempt because they do not use the shared scalar resource.
1124 if (HII.isHLXVec(MI) && HII.isHLXVec(MI: MJ) && !MJ.mayLoad() && !MJ.mayStore()) {
1125 auto getScalarUses = [](const MachineInstr &MI) {
1126 unsigned Uses = 0;
1127 for (const MachineOperand &MO : MI.explicit_uses()) {
1128 if (MO.isImm())
1129 Uses |= 1;
1130 else if (MO.isReg() && Hexagon::IntRegsRegClass.contains(Reg: MO.getReg()))
1131 Uses |= 2;
1132 else if (MO.isReg() &&
1133 Hexagon::DoubleRegsRegClass.contains(Reg: MO.getReg()))
1134 Uses |= 4;
1135 }
1136 return Uses;
1137 };
1138
1139 if ((getScalarUses(MI) & 4) && getScalarUses(MJ))
1140 return true;
1141 }
1142
1143 switch (MI.getOpcode()) {
1144 case Hexagon::S2_storew_locked:
1145 case Hexagon::S4_stored_locked:
1146 case Hexagon::L2_loadw_locked:
1147 case Hexagon::L4_loadd_locked:
1148 case Hexagon::Y2_dccleana:
1149 case Hexagon::Y2_dccleaninva:
1150 case Hexagon::Y2_dcinva:
1151 case Hexagon::Y2_dczeroa:
1152 case Hexagon::Y4_l2fetch:
1153 case Hexagon::Y5_l2fetch: {
1154 // These instructions can only be grouped with ALU32 or non-floating-point
1155 // XTYPE instructions. Since there is no convenient way of identifying fp
1156 // XTYPE instructions, only allow grouping with ALU32 for now.
1157 unsigned TJ = HII.getType(MI: MJ);
1158 if (TJ != HexagonII::TypeALU32_2op &&
1159 TJ != HexagonII::TypeALU32_3op &&
1160 TJ != HexagonII::TypeALU32_ADDI)
1161 return true;
1162 break;
1163 }
1164 case Hexagon::LX_S16b_pseudo: {
1165 // The store pseudo expands to a copy of the stored register followed by a
1166 // new-value store. Do not packetize it with an instruction defining that
1167 // register, otherwise the packet would have two definitions of it.
1168 Register StReg = MI.getOperand(i: 1).getReg();
1169 const HexagonRegisterInfo &HRI = HII.getRegisterInfo();
1170 if (any_of(Range: MJ.all_defs(), P: [&](const MachineOperand &Def) {
1171 return HRI.isSuperRegisterEq(RegA: StReg, RegB: Def.getReg());
1172 }))
1173 return true;
1174 break;
1175 }
1176 default:
1177 break;
1178 }
1179
1180 // "False" really means that the quick check failed to determine if
1181 // I and J cannot coexist.
1182 return false;
1183}
1184
1185// Full, symmetric check.
1186bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI,
1187 const MachineInstr &MJ) {
1188 return cannotCoexistAsymm(MI, MJ, HII: *HII) || cannotCoexistAsymm(MI: MJ, MJ: MI, HII: *HII);
1189}
1190
1191void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
1192 for (auto &B : MF) {
1193 MachineBasicBlock::iterator BundleIt;
1194 for (MachineInstr &MI : llvm::make_early_inc_range(Range: B.instrs())) {
1195 if (MI.isBundle())
1196 BundleIt = MI.getIterator();
1197 if (!MI.isInsideBundle())
1198 continue;
1199
1200 // Decide on where to insert the instruction that we are pulling out.
1201 // Debug instructions always go before the bundle, but the placement of
1202 // INLINE_ASM depends on potential dependencies. By default, try to
1203 // put it before the bundle, but if the asm writes to a register that
1204 // other instructions in the bundle read, then we need to place it
1205 // after the bundle (to preserve the bundle semantics).
1206 bool InsertBeforeBundle;
1207 if (MI.isInlineAsm())
1208 InsertBeforeBundle = !hasWriteToReadDep(FirstI: MI, SecondI: *BundleIt, TRI: HRI);
1209 else if (MI.isDebugInstr())
1210 InsertBeforeBundle = true;
1211 else
1212 continue;
1213
1214 BundleIt = moveInstrOut(MI, BundleIt, Before: InsertBeforeBundle);
1215 }
1216 }
1217}
1218
1219// Check if a given instruction is of class "system".
1220static bool isSystemInstr(const MachineInstr &MI) {
1221 unsigned Opc = MI.getOpcode();
1222 switch (Opc) {
1223 case Hexagon::Y2_barrier:
1224 case Hexagon::Y2_dcfetchbo:
1225 case Hexagon::Y4_l2fetch:
1226 case Hexagon::Y5_l2fetch:
1227 return true;
1228 }
1229 return false;
1230}
1231
1232bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I,
1233 const MachineInstr &J) {
1234 // The dependence graph may not include edges between dead definitions,
1235 // so without extra checks, we could end up packetizing two instruction
1236 // defining the same (dead) register.
1237 if (I.isCall() || J.isCall())
1238 return false;
1239 if (HII->isPredicated(MI: I) || HII->isPredicated(MI: J))
1240 return false;
1241
1242 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
1243 for (auto &MO : I.operands()) {
1244 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1245 continue;
1246 DeadDefs[MO.getReg()] = true;
1247 }
1248
1249 for (auto &MO : J.operands()) {
1250 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1251 continue;
1252 Register R = MO.getReg();
1253 if (R != Hexagon::USR_OVF && DeadDefs[R])
1254 return true;
1255 }
1256 return false;
1257}
1258
1259bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I,
1260 const MachineInstr &J) {
1261 // A save callee-save register function call can only be in a packet
1262 // with instructions that don't write to the callee-save registers.
1263 if ((HII->isSaveCalleeSavedRegsCall(MI: I) &&
1264 doesModifyCalleeSavedReg(MI: J, TRI: HRI)) ||
1265 (HII->isSaveCalleeSavedRegsCall(MI: J) &&
1266 doesModifyCalleeSavedReg(MI: I, TRI: HRI)))
1267 return true;
1268
1269 // Two control flow instructions cannot go in the same packet.
1270 if (isControlFlow(MI: I) && isControlFlow(MI: J))
1271 return true;
1272
1273 // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1274 // contain a speculative indirect jump,
1275 // a new-value compare jump or a dealloc_return.
1276 auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
1277 if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
1278 return true;
1279 if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
1280 return true;
1281 return false;
1282 };
1283
1284 if (HII->isLoopN(MI: I) && isBadForLoopN(J))
1285 return true;
1286 if (HII->isLoopN(MI: J) && isBadForLoopN(I))
1287 return true;
1288
1289 // dealloc_return cannot appear in the same packet as a conditional or
1290 // unconditional jump.
1291 return HII->isDeallocRet(MI: I) &&
1292 (J.isBranch() || J.isCall() || J.isBarrier());
1293}
1294
1295bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I,
1296 const MachineInstr &J) {
1297 // Adding I to a packet that has J.
1298
1299 // Regmasks are not reflected in the scheduling dependency graph, so
1300 // we need to check them manually. This code assumes that regmasks only
1301 // occur on calls, and the problematic case is when we add an instruction
1302 // defining a register R to a packet that has a call that clobbers R via
1303 // a regmask. Those cannot be packetized together, because the call will
1304 // be executed last. That's also a reason why it is ok to add a call
1305 // clobbering R to a packet that defines R.
1306
1307 // Look for regmasks in J.
1308 for (const MachineOperand &OpJ : J.operands()) {
1309 if (!OpJ.isRegMask())
1310 continue;
1311 assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");
1312 for (const MachineOperand &OpI : I.operands()) {
1313 if (OpI.isReg()) {
1314 if (OpJ.clobbersPhysReg(PhysReg: OpI.getReg()))
1315 return true;
1316 } else if (OpI.isRegMask()) {
1317 // Both are regmasks. Assume that they intersect.
1318 return true;
1319 }
1320 }
1321 }
1322 return false;
1323}
1324
1325bool HexagonPacketizerList::hasDualStoreDependence(const MachineInstr &I,
1326 const MachineInstr &J) {
1327 bool SysI = isSystemInstr(MI: I), SysJ = isSystemInstr(MI: J);
1328 bool StoreI = I.mayStore(), StoreJ = J.mayStore();
1329 if ((SysI && StoreJ) || (SysJ && StoreI))
1330 return true;
1331
1332 if (StoreI && StoreJ) {
1333 if (HII->isNewValueInst(MI: J) || HII->isMemOp(MI: J) || HII->isMemOp(MI: I))
1334 return true;
1335 } else {
1336 // A memop cannot be in the same packet with another memop or a store.
1337 // Two stores can be together, but here I and J cannot both be stores.
1338 bool MopStI = HII->isMemOp(MI: I) || StoreI;
1339 bool MopStJ = HII->isMemOp(MI: J) || StoreJ;
1340 if (MopStI && MopStJ)
1341 return true;
1342 }
1343
1344 return (StoreJ && HII->isDeallocRet(MI: I)) || (StoreI && HII->isDeallocRet(MI: J));
1345}
1346
1347// SUI is the current instruction that is outside of the current packet.
1348// SUJ is the current instruction inside the current packet against which that
1349// SUI will be packetized.
1350bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
1351 assert(SUI->getInstr() && SUJ->getInstr());
1352 MachineInstr &I = *SUI->getInstr();
1353 MachineInstr &J = *SUJ->getInstr();
1354
1355 // Clear IgnoreDepMIs when Packet starts.
1356 if (CurrentPacketMIs.size() == 1)
1357 IgnoreDepMIs.clear();
1358
1359 MachineBasicBlock::iterator II = I.getIterator();
1360
1361 // Solo instructions cannot go in the packet.
1362 assert(!isSoloInstruction(I) && "Unexpected solo instr!");
1363
1364 if (cannotCoexist(MI: I, MJ: J))
1365 return false;
1366
1367 Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1368 if (Dependence)
1369 return false;
1370
1371 // Regmasks are not accounted for in the scheduling graph, so we need
1372 // to explicitly check for dependencies caused by them. They should only
1373 // appear on calls, so it's not too pessimistic to reject all regmask
1374 // dependencies.
1375 Dependence = hasRegMaskDependence(I, J);
1376 if (Dependence)
1377 return false;
1378
1379 // Dual-store does not allow second store, if the first store is not
1380 // in SLOT0. New value store, new value jump, dealloc_return and memop
1381 // always take SLOT0. Arch spec 3.4.4.2.
1382 Dependence = hasDualStoreDependence(I, J);
1383 if (Dependence)
1384 return false;
1385
1386 // If an instruction feeds new value jump, glue it.
1387 MachineBasicBlock::iterator NextMII = I.getIterator();
1388 ++NextMII;
1389 if (NextMII != I.getParent()->end() && HII->isNewValueJump(MI: *NextMII)) {
1390 MachineInstr &NextMI = *NextMII;
1391
1392 bool secondRegMatch = false;
1393 const MachineOperand &NOp0 = NextMI.getOperand(i: 0);
1394 const MachineOperand &NOp1 = NextMI.getOperand(i: 1);
1395
1396 if (NOp1.isReg() && I.getOperand(i: 0).getReg() == NOp1.getReg())
1397 secondRegMatch = true;
1398
1399 for (MachineInstr *PI : CurrentPacketMIs) {
1400 // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
1401 if (PI->isCall()) {
1402 Dependence = true;
1403 break;
1404 }
1405 // Validate:
1406 // 1. Packet does not have a store in it.
1407 // 2. If the first operand of the nvj is newified, and the second
1408 // operand is also a reg, it (second reg) is not defined in
1409 // the same packet.
1410 // 3. If the second operand of the nvj is newified, (which means
1411 // first operand is also a reg), first reg is not defined in
1412 // the same packet.
1413 if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
1414 HII->isLoopN(MI: *PI)) {
1415 Dependence = true;
1416 break;
1417 }
1418 // Check #2/#3.
1419 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
1420 if (OpR.isReg() && PI->modifiesRegister(Reg: OpR.getReg(), TRI: HRI)) {
1421 Dependence = true;
1422 break;
1423 }
1424 }
1425
1426 GlueToNewValueJump = true;
1427 if (Dependence)
1428 return false;
1429 }
1430
1431 // There no dependency between a prolog instruction and its successor.
1432 if (!SUJ->isSucc(N: SUI))
1433 return true;
1434
1435 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1436 if (FoundSequentialDependence)
1437 break;
1438
1439 if (SUJ->Succs[i].getSUnit() != SUI)
1440 continue;
1441
1442 SDep::Kind DepType = SUJ->Succs[i].getKind();
1443 // For direct calls:
1444 // Ignore register dependences for call instructions for packetization
1445 // purposes except for those due to r31 and predicate registers.
1446 //
1447 // For indirect calls:
1448 // Same as direct calls + check for true dependences to the register
1449 // used in the indirect call.
1450 //
1451 // We completely ignore Order dependences for call instructions.
1452 //
1453 // For returns:
1454 // Ignore register dependences for return instructions like jumpr,
1455 // dealloc return unless we have dependencies on the explicit uses
1456 // of the registers used by jumpr (like r31) or dealloc return
1457 // (like r29 or r30).
1458 unsigned DepReg = 0;
1459 const TargetRegisterClass *RC = nullptr;
1460 if (DepType == SDep::Data) {
1461 DepReg = SUJ->Succs[i].getReg();
1462 RC = HRI->getMinimalPhysRegClass(Reg: DepReg);
1463 }
1464
1465 if (I.isCall() || HII->isJumpR(MI: I) || I.isReturn() || HII->isTailCall(MI: I)) {
1466 if (!isRegDependence(DepType))
1467 continue;
1468 if (!isCallDependent(MI: I, DepType, DepReg: SUJ->Succs[i].getReg()))
1469 continue;
1470 }
1471
1472 if (DepType == SDep::Data) {
1473 if (canPromoteToDotCur(MI: J, PacketSU: SUJ, DepReg, MII&: II, RC))
1474 if (promoteToDotCur(MI&: J, DepType, MII&: II, RC))
1475 continue;
1476 }
1477
1478 // Data dependence ok if we have load.cur.
1479 if (DepType == SDep::Data && HII->isDotCurInst(MI: J)) {
1480 if (HII->isHVXVec(MI: I))
1481 continue;
1482 }
1483
1484 // For instructions that can be promoted to dot-new, try to promote.
1485 if (DepType == SDep::Data) {
1486 if (canPromoteToDotNew(MI: I, PacketSU: SUJ, DepReg, MII&: II, RC)) {
1487 if (promoteToDotNew(MI&: I, DepType, MII&: II, RC)) {
1488 PromotedToDotNew = true;
1489 if (cannotCoexist(MI: I, MJ: J))
1490 FoundSequentialDependence = true;
1491 continue;
1492 }
1493 }
1494 if (HII->isNewValueJump(MI: I))
1495 continue;
1496 }
1497
1498 // For predicated instructions, if the predicates are complements then
1499 // there can be no dependence.
1500 if (HII->isPredicated(MI: I) && HII->isPredicated(MI: J) &&
1501 arePredicatesComplements(MI1&: I, MI2&: J)) {
1502 // Not always safe to do this translation.
1503 // DAG Builder attempts to reduce dependence edges using transitive
1504 // nature of dependencies. Here is an example:
1505 //
1506 // r0 = tfr_pt ... (1)
1507 // r0 = tfr_pf ... (2)
1508 // r0 = tfr_pt ... (3)
1509 //
1510 // There will be an output dependence between (1)->(2) and (2)->(3).
1511 // However, there is no dependence edge between (1)->(3). This results
1512 // in all 3 instructions going in the same packet. We ignore dependce
1513 // only once to avoid this situation.
1514 auto Itr = find(Range&: IgnoreDepMIs, Val: &J);
1515 if (Itr != IgnoreDepMIs.end()) {
1516 Dependence = true;
1517 return false;
1518 }
1519 IgnoreDepMIs.push_back(x: &I);
1520 continue;
1521 }
1522
1523 // Ignore Order dependences between unconditional direct branches
1524 // and non-control-flow instructions.
1525 if (isDirectJump(MI: I) && !J.isBranch() && !J.isCall() &&
1526 DepType == SDep::Order)
1527 continue;
1528
1529 // Ignore all dependences for jumps except for true and output
1530 // dependences.
1531 if (I.isConditionalBranch() && DepType != SDep::Data &&
1532 DepType != SDep::Output)
1533 continue;
1534
1535 if (DepType == SDep::Output) {
1536 FoundSequentialDependence = true;
1537 break;
1538 }
1539
1540 // For Order dependences:
1541 // 1. Volatile loads/stores can be packetized together, unless other
1542 // rules prevent is.
1543 // 2. Store followed by a load is not allowed.
1544 // 3. Store followed by a store is valid.
1545 // 4. Load followed by any memory operation is allowed.
1546 if (DepType == SDep::Order) {
1547 if (!PacketizeVolatiles) {
1548 bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
1549 if (OrdRefs) {
1550 FoundSequentialDependence = true;
1551 break;
1552 }
1553 }
1554 // J is first, I is second.
1555 bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
1556 bool LoadI = I.mayLoad(), StoreI = I.mayStore();
1557 bool NVStoreJ = HII->isNewValueStore(MI: J);
1558 bool NVStoreI = HII->isNewValueStore(MI: I);
1559 bool IsVecJ = HII->isHVXVec(MI: J) || HII->isHLXVec(MI: J);
1560 bool IsVecI = HII->isHVXVec(MI: I) || HII->isHLXVec(MI: I);
1561
1562 // Don't reorder the loads if there is an order dependence. This would
1563 // occur if the first instruction must go in slot0.
1564 if (LoadJ && LoadI && HII->isPureSlot0(MI: J)) {
1565 FoundSequentialDependence = true;
1566 break;
1567 }
1568
1569 if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() &&
1570 ((LoadJ && StoreI && !NVStoreI) ||
1571 (StoreJ && LoadI && !NVStoreJ)) &&
1572 (J.getOpcode() != Hexagon::S2_allocframe &&
1573 I.getOpcode() != Hexagon::S2_allocframe) &&
1574 (J.getOpcode() != Hexagon::L2_deallocframe &&
1575 I.getOpcode() != Hexagon::L2_deallocframe) &&
1576 (!HII->isMemOp(MI: J) && !HII->isMemOp(MI: I)) && (!IsVecJ && !IsVecI))
1577 setmemShufDisabled(true);
1578 else
1579 if (StoreJ && LoadI && alias(MI1: J, MI2: I)) {
1580 FoundSequentialDependence = true;
1581 break;
1582 }
1583
1584 if (!StoreJ)
1585 if (!LoadJ || (!LoadI && !StoreI)) {
1586 // If J is neither load nor store, assume a dependency.
1587 // If J is a load, but I is neither, also assume a dependency.
1588 FoundSequentialDependence = true;
1589 break;
1590 }
1591 // Store followed by store: not OK on V2.
1592 // Store followed by load: not OK on all.
1593 // Load followed by store: OK on all.
1594 // Load followed by load: OK on all.
1595 continue;
1596 }
1597
1598 // Special case for ALLOCFRAME: even though there is dependency
1599 // between ALLOCFRAME and subsequent store, allow it to be packetized
1600 // in a same packet. This implies that the store is using the caller's
1601 // SP. Hence, offset needs to be updated accordingly.
1602 if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
1603 unsigned Opc = I.getOpcode();
1604 switch (Opc) {
1605 case Hexagon::S2_storerd_io:
1606 case Hexagon::S2_storeri_io:
1607 case Hexagon::S2_storerh_io:
1608 case Hexagon::S2_storerb_io:
1609 if (I.getOperand(i: 0).getReg() == HRI->getStackRegister()) {
1610 // Since this store is to be glued with allocframe in the same
1611 // packet, it will use SP of the previous stack frame, i.e.
1612 // caller's SP. Therefore, we need to recalculate offset
1613 // according to this change.
1614 GlueAllocframeStore = useCallersSP(MI&: I);
1615 if (GlueAllocframeStore)
1616 continue;
1617 }
1618 break;
1619 default:
1620 break;
1621 }
1622 }
1623
1624 // There are certain anti-dependencies that cannot be ignored.
1625 // Specifically:
1626 // J2_call ... implicit-def %r0 ; SUJ
1627 // R0 = ... ; SUI
1628 // Those cannot be packetized together, since the call will observe
1629 // the effect of the assignment to R0.
1630 if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) {
1631 // Check if I defines any volatile register. We should also check
1632 // registers that the call may read, but these happen to be a
1633 // subset of the volatile register set.
1634 for (const MachineOperand &Op : I.operands()) {
1635 if (Op.isReg() && Op.isDef()) {
1636 Register R = Op.getReg();
1637 if (!J.readsRegister(Reg: R, TRI: HRI) && !J.modifiesRegister(Reg: R, TRI: HRI))
1638 continue;
1639 } else if (!Op.isRegMask()) {
1640 // If I has a regmask assume dependency.
1641 continue;
1642 }
1643 FoundSequentialDependence = true;
1644 break;
1645 }
1646 }
1647
1648 // Skip over remaining anti-dependences. Two instructions that are
1649 // anti-dependent can share a packet, since in most such cases all
1650 // operands are read before any modifications take place.
1651 // The exceptions are branch and call instructions, since they are
1652 // executed after all other instructions have completed (at least
1653 // conceptually).
1654 // Another exception is the HLX load pseudo, which expands to a temporary
1655 // load and a copy of the loaded value. An instruction it is anti-dependent
1656 // on would read the new value if both were in the same packet.
1657 if (DepType != SDep::Anti || I.getOpcode() == Hexagon::LX_L16b_pseudo) {
1658 FoundSequentialDependence = true;
1659 break;
1660 }
1661 }
1662
1663 if (FoundSequentialDependence) {
1664 Dependence = true;
1665 return false;
1666 }
1667
1668 return true;
1669}
1670
1671bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
1672 assert(SUI->getInstr() && SUJ->getInstr());
1673 MachineInstr &I = *SUI->getInstr();
1674 MachineInstr &J = *SUJ->getInstr();
1675
1676 bool Coexist = !cannotCoexist(MI: I, MJ: J);
1677
1678 if (Coexist && !Dependence)
1679 return true;
1680
1681 // Check if the instruction was promoted to a dot-new. If so, demote it
1682 // back into a dot-old.
1683 if (PromotedToDotNew)
1684 demoteToDotOld(MI&: I);
1685
1686 cleanUpDotCur();
1687 // Check if the instruction (must be a store) was glued with an allocframe
1688 // instruction. If so, restore its offset to its original value, i.e. use
1689 // current SP instead of caller's SP.
1690 if (GlueAllocframeStore) {
1691 useCalleesSP(MI&: I);
1692 GlueAllocframeStore = false;
1693 }
1694
1695 if (ChangedOffset != INT64_MAX)
1696 undoChangedOffset(MI&: I);
1697
1698 if (GlueToNewValueJump) {
1699 // Putting I and J together would prevent the new-value jump from being
1700 // packetized with the producer. In that case I and J must be separated.
1701 GlueToNewValueJump = false;
1702 return false;
1703 }
1704
1705 if (!Coexist)
1706 return false;
1707
1708 if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) {
1709 FoundSequentialDependence = false;
1710 Dependence = false;
1711 return true;
1712 }
1713
1714 return false;
1715}
1716
1717
1718bool HexagonPacketizerList::foundLSInPacket() {
1719 bool FoundLoad = false;
1720 bool FoundStore = false;
1721
1722 for (auto *MJ : CurrentPacketMIs) {
1723 unsigned Opc = MJ->getOpcode();
1724 if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe)
1725 continue;
1726 if (HII->isMemOp(MI: *MJ))
1727 continue;
1728 if (MJ->mayLoad())
1729 FoundLoad = true;
1730 if (MJ->mayStore() && !HII->isNewValueStore(MI: *MJ))
1731 FoundStore = true;
1732 }
1733 return FoundLoad && FoundStore;
1734}
1735
1736
1737MachineBasicBlock::iterator
1738HexagonPacketizerList::addToPacket(MachineInstr &MI) {
1739 MachineBasicBlock::iterator MII = MI.getIterator();
1740 MachineBasicBlock *MBB = MI.getParent();
1741
1742 if (CurrentPacketMIs.empty()) {
1743 PacketStalls = false;
1744 PacketStallCycles = 0;
1745 }
1746 PacketStalls |= producesStall(MI);
1747 PacketStallCycles = std::max(a: PacketStallCycles, b: calcStall(MI));
1748
1749 if (MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::FREEZE) {
1750 // Add to the packet to allow subsequent instructions to be checked
1751 // properly.
1752 CurrentPacketMIs.push_back(x: &MI);
1753 return MII;
1754 }
1755 assert(ResourceTracker->canReserveResources(MI));
1756
1757 MachineBasicBlock::iterator nextMII = MII;
1758 if (std::next(x: MII) != MI.getParent()->end()) {
1759 MachineInstr &nextMI = *++nextMII;
1760 if (HII->isActivateWeightPair(MIa: MI, MIb: nextMI)) {
1761 for (auto J : CurrentPacketMIs) {
1762 MachineInstr &JMI = *J;
1763 if (HII->isDependent(ProdMI: JMI, ConsMI: MI) || HII->isDependent(ProdMI: JMI, ConsMI: nextMI)) {
1764 endPacket(MBB, MI);
1765 break;
1766 }
1767 }
1768 ResourceTracker->reserveResources(MI);
1769 if (ResourceTracker->canReserveResources(MI&: nextMI))
1770 ResourceTracker->reserveResources(MI&: nextMI);
1771 else {
1772 endPacket(MBB, MI);
1773 assert(ResourceTracker->canReserveResources(MI));
1774 ResourceTracker->reserveResources(MI);
1775 assert(ResourceTracker->canReserveResources(nextMI));
1776 ResourceTracker->reserveResources(MI&: nextMI);
1777 }
1778 CurrentPacketMIs.push_back(x: &MI);
1779 CurrentPacketMIs.push_back(x: &nextMI);
1780 return nextMII;
1781 }
1782 }
1783
1784 bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
1785 bool Good = true;
1786
1787 if (GlueToNewValueJump) {
1788 MachineInstr &NvjMI = *++MII;
1789 // We need to put both instructions in the same packet: MI and NvjMI.
1790 // Either of them can require a constant extender. Try to add both to
1791 // the current packet, and if that fails, end the packet and start a
1792 // new one.
1793 ResourceTracker->reserveResources(MI);
1794 if (ExtMI)
1795 Good = tryAllocateResourcesForConstExt(Reserve: true);
1796
1797 bool ExtNvjMI = HII->isExtended(MI: NvjMI) || HII->isConstExtended(MI: NvjMI);
1798 if (Good) {
1799 if (ResourceTracker->canReserveResources(MI&: NvjMI))
1800 ResourceTracker->reserveResources(MI&: NvjMI);
1801 else
1802 Good = false;
1803 }
1804 if (Good && ExtNvjMI)
1805 Good = tryAllocateResourcesForConstExt(Reserve: true);
1806
1807 if (!Good) {
1808 endPacket(MBB, MI);
1809 assert(ResourceTracker->canReserveResources(MI));
1810 ResourceTracker->reserveResources(MI);
1811 if (ExtMI) {
1812 assert(canReserveResourcesForConstExt());
1813 tryAllocateResourcesForConstExt(Reserve: true);
1814 }
1815 assert(ResourceTracker->canReserveResources(NvjMI));
1816 ResourceTracker->reserveResources(MI&: NvjMI);
1817 if (ExtNvjMI) {
1818 assert(canReserveResourcesForConstExt());
1819 reserveResourcesForConstExt();
1820 }
1821 }
1822 CurrentPacketMIs.push_back(x: &MI);
1823 CurrentPacketMIs.push_back(x: &NvjMI);
1824 return MII;
1825 }
1826
1827 ResourceTracker->reserveResources(MI);
1828 if (ExtMI && !tryAllocateResourcesForConstExt(Reserve: true)) {
1829 endPacket(MBB, MI);
1830 if (PromotedToDotNew)
1831 demoteToDotOld(MI);
1832 if (GlueAllocframeStore) {
1833 useCalleesSP(MI);
1834 GlueAllocframeStore = false;
1835 }
1836 ResourceTracker->reserveResources(MI);
1837 reserveResourcesForConstExt();
1838 }
1839
1840 CurrentPacketMIs.push_back(x: &MI);
1841 return MII;
1842}
1843
1844void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
1845 MachineBasicBlock::iterator EndMI) {
1846 // Replace VLIWPacketizerList::endPacket(MBB, EndMI).
1847 LLVM_DEBUG({
1848 if (!CurrentPacketMIs.empty()) {
1849 dbgs() << "Finalizing packet:\n";
1850 unsigned Idx = 0;
1851 for (MachineInstr *MI : CurrentPacketMIs) {
1852 unsigned R = ResourceTracker->getUsedResources(Idx++);
1853 dbgs() << " * [res:0x" << utohexstr(R) << "] " << *MI;
1854 }
1855 }
1856 });
1857
1858 bool memShufDisabled = getmemShufDisabled();
1859 if (memShufDisabled && !foundLSInPacket()) {
1860 setmemShufDisabled(false);
1861 LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n");
1862 }
1863 memShufDisabled = getmemShufDisabled();
1864
1865 OldPacketMIs.clear();
1866 for (MachineInstr *MI : CurrentPacketMIs) {
1867 MachineBasicBlock::instr_iterator NextMI = std::next(x: MI->getIterator());
1868 for (auto &I : make_range(x: HII->expandVGatherPseudo(MI&: *MI), y: NextMI))
1869 OldPacketMIs.push_back(x: &I);
1870 }
1871 CurrentPacketMIs.clear();
1872
1873 if (OldPacketMIs.size() > 1) {
1874 MachineBasicBlock::instr_iterator FirstMI(OldPacketMIs.front());
1875 MachineBasicBlock::instr_iterator LastMI(EndMI.getInstrIterator());
1876 finalizeBundle(MBB&: *MBB, FirstMI, LastMI);
1877 auto BundleMII = std::prev(x: FirstMI);
1878 if (memShufDisabled)
1879 HII->setBundleNoShuf(BundleMII);
1880
1881 setmemShufDisabled(false);
1882 }
1883
1884 PacketHasDuplex = false;
1885 PacketHasSLOT0OnlyInsn = false;
1886 ResourceTracker->clearResources();
1887 LLVM_DEBUG(dbgs() << "End packet\n");
1888}
1889
1890bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
1891 if (Minimal)
1892 return false;
1893
1894 if (producesStall(MI))
1895 return false;
1896
1897 // If TinyCore with Duplexes is enabled, check if this MI can form a Duplex
1898 // with any other instruction in the existing packet.
1899 auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>();
1900 // Constraint 1: Only one duplex allowed per packet.
1901 // Constraint 2: Consider duplex checks only if there is at least one
1902 // instruction in a packet.
1903 // Constraint 3: If one of the existing instructions in the packet has a
1904 // SLOT0 only instruction that can not be duplexed, do not attempt to form
1905 // duplexes. (TODO: This will invalidate the L4_return* instructions to form a
1906 // duplex)
1907 if (HST.isTinyCoreWithDuplex() && CurrentPacketMIs.size() > 0 &&
1908 !PacketHasDuplex) {
1909 // Check for SLOT0 only non-duplexable instruction in packet.
1910 for (auto &MJ : CurrentPacketMIs)
1911 PacketHasSLOT0OnlyInsn |= HII->isPureSlot0(MI: *MJ);
1912 // Get the Big Core Opcode (dup_*).
1913 int Opcode = HII->getDuplexOpcode(MI, ForBigCore: false);
1914 if (Opcode >= 0) {
1915 // We now have an instruction that can be duplexed.
1916 for (auto &MJ : CurrentPacketMIs) {
1917 if (HII->isDuplexPair(MIa: MI, MIb: *MJ) && !PacketHasSLOT0OnlyInsn) {
1918 PacketHasDuplex = true;
1919 return true;
1920 }
1921 }
1922 // If it can not be duplexed, check if there is a valid transition in DFA
1923 // with the original opcode.
1924 MachineInstr &MIRef = const_cast<MachineInstr &>(MI);
1925 MIRef.setDesc(HII->get(Opcode));
1926 return ResourceTracker->canReserveResources(MI&: MIRef);
1927 }
1928 }
1929
1930 // An HLX load/store pseudo expands to two instructions in its packet. With
1931 // one more instruction in the packet, there is no room left for a
1932 // constant-extended instruction and its extender within the four-word
1933 // packet limit.
1934 if (HST.useHLXOps() && CurrentPacketMIs.size() > 1) {
1935 auto IsHLXLdStPseudo = [this](const MachineInstr &I) {
1936 return HII->isTmpLoadPseudo(MI: I) || HII->isNewStorePseudo(MI: I);
1937 };
1938 bool HasConstExt = HII->isConstExtended(MI);
1939 bool HasPseudo = IsHLXLdStPseudo(MI);
1940 for (const MachineInstr *MJ : CurrentPacketMIs) {
1941 HasConstExt |= HII->isConstExtended(MI: *MJ);
1942 HasPseudo |= IsHLXLdStPseudo(*MJ);
1943 }
1944 if (HasConstExt && HasPseudo)
1945 return false;
1946 }
1947
1948 return true;
1949}
1950
1951// V60 forward scheduling.
1952unsigned int HexagonPacketizerList::calcStall(const MachineInstr &I) {
1953 // Check whether the previous packet is in a different loop. If this is the
1954 // case, there is little point in trying to avoid a stall because that would
1955 // favor the rare case (loop entry) over the common case (loop iteration).
1956 //
1957 // TODO: We should really be able to check all the incoming edges if this is
1958 // the first packet in a basic block, so we can avoid stalls from the loop
1959 // backedge.
1960 if (!OldPacketMIs.empty()) {
1961 auto *OldBB = OldPacketMIs.front()->getParent();
1962 auto *ThisBB = I.getParent();
1963 if (MLI->getLoopFor(BB: OldBB) != MLI->getLoopFor(BB: ThisBB))
1964 return 0;
1965 }
1966
1967 SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)];
1968 if (!SUI)
1969 return 0;
1970
1971 // If the latency is 0 and there is a data dependence between this
1972 // instruction and any instruction in the current packet, we disregard any
1973 // potential stalls due to the instructions in the previous packet. Most of
1974 // the instruction pairs that can go together in the same packet have 0
1975 // latency between them. The exceptions are
1976 // 1. NewValueJumps as they're generated much later and the latencies can't
1977 // be changed at that point.
1978 // 2. .cur instructions, if its consumer has a 0 latency successor (such as
1979 // .new). In this case, the latency between .cur and the consumer stays
1980 // non-zero even though we can have both .cur and .new in the same packet.
1981 // Changing the latency to 0 is not an option as it causes software pipeliner
1982 // to not pipeline in some cases.
1983
1984 // For Example:
1985 // {
1986 // I1: v6.cur = vmem(r0++#1)
1987 // I2: v7 = valign(v6,v4,r2)
1988 // I3: vmem(r5++#1) = v7.new
1989 // }
1990 // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2.
1991
1992 for (auto *J : CurrentPacketMIs) {
1993 SUnit *SUJ = MIToSUnit[J];
1994 for (auto &Pred : SUI->Preds)
1995 if (Pred.getSUnit() == SUJ)
1996 if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) ||
1997 HII->isNewValueJump(MI: I) || HII->isToBeScheduledASAP(MI1: *J, MI2: I))
1998 return 0;
1999 }
2000
2001 // Check if the latency is greater than one between this instruction and any
2002 // instruction in the previous packet.
2003 for (auto *J : OldPacketMIs) {
2004 SUnit *SUJ = MIToSUnit[J];
2005 for (auto &Pred : SUI->Preds)
2006 if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
2007 return Pred.getLatency();
2008 }
2009
2010 return 0;
2011}
2012
2013bool HexagonPacketizerList::producesStall(const MachineInstr &I) {
2014 unsigned int Latency = calcStall(I);
2015 if (Latency == 0)
2016 return false;
2017 // Ignore stall unless it stalls more than previous instruction in packet
2018 if (PacketStalls)
2019 return Latency > PacketStallCycles;
2020 return true;
2021}
2022
2023//===----------------------------------------------------------------------===//
2024// Public Constructor Functions
2025//===----------------------------------------------------------------------===//
2026
2027FunctionPass *llvm::createHexagonPacketizer(bool Minimal) {
2028 return new HexagonPacketizer(Minimal);
2029}
2030