| 1 | //===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This implements a simple VLIW packetizer using DFA. The packetizer works on |
| 10 | // machine basic blocks. For each instruction I in BB, the packetizer consults |
| 11 | // the DFA to see if machine resources are available to execute I. If so, the |
| 12 | // packetizer checks if I depends on any instruction J in the current packet. |
| 13 | // If no dependency is found, I is added to current packet and machine resource |
| 14 | // is marked as taken. If any dependency is found, a target API call is made to |
| 15 | // prune the dependence. |
| 16 | // |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | |
| 19 | #include "HexagonVLIWPacketizer.h" |
| 20 | #include "Hexagon.h" |
| 21 | #include "HexagonInstrInfo.h" |
| 22 | #include "HexagonRegisterInfo.h" |
| 23 | #include "HexagonSubtarget.h" |
| 24 | #include "llvm/ADT/BitVector.h" |
| 25 | #include "llvm/ADT/DenseSet.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/StringExtras.h" |
| 28 | #include "llvm/Analysis/AliasAnalysis.h" |
| 29 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 30 | #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" |
| 31 | #include "llvm/CodeGen/MachineDominators.h" |
| 32 | #include "llvm/CodeGen/MachineFrameInfo.h" |
| 33 | #include "llvm/CodeGen/MachineFunction.h" |
| 34 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 35 | #include "llvm/CodeGen/MachineInstr.h" |
| 36 | #include "llvm/CodeGen/MachineInstrBundle.h" |
| 37 | #include "llvm/CodeGen/MachineLoopInfo.h" |
| 38 | #include "llvm/CodeGen/MachineOperand.h" |
| 39 | #include "llvm/CodeGen/ScheduleDAG.h" |
| 40 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
| 41 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
| 42 | #include "llvm/IR/DebugLoc.h" |
| 43 | #include "llvm/InitializePasses.h" |
| 44 | #include "llvm/MC/MCInstrDesc.h" |
| 45 | #include "llvm/Pass.h" |
| 46 | #include "llvm/Support/CommandLine.h" |
| 47 | #include "llvm/Support/Debug.h" |
| 48 | #include "llvm/Support/ErrorHandling.h" |
| 49 | #include "llvm/Support/raw_ostream.h" |
| 50 | #include <cassert> |
| 51 | #include <cstdint> |
| 52 | #include <iterator> |
| 53 | |
| 54 | using namespace llvm; |
| 55 | |
| 56 | #define DEBUG_TYPE "packets" |
| 57 | |
| 58 | cl::opt<bool> DisablePacketizer("disable-packetizer" , cl::Hidden, |
| 59 | cl::desc("Disable Hexagon packetizer pass" )); |
| 60 | |
| 61 | static cl::opt<bool> Slot1Store("slot1-store-slot0-load" , cl::Hidden, |
| 62 | cl::init(Val: true), |
| 63 | cl::desc("Allow slot1 store and slot0 load" )); |
| 64 | |
| 65 | static cl::opt<bool> PacketizeVolatiles( |
| 66 | "hexagon-packetize-volatiles" , cl::Hidden, cl::init(Val: true), |
| 67 | cl::desc("Allow non-solo packetization of volatile memory references" )); |
| 68 | |
| 69 | static cl::opt<bool> |
| 70 | EnableGenAllInsnClass("enable-gen-insn" , cl::Hidden, |
| 71 | cl::desc("Generate all instruction with TC" )); |
| 72 | |
| 73 | static cl::opt<bool> |
| 74 | DisableVecDblNVStores("disable-vecdbl-nv-stores" , cl::Hidden, |
| 75 | cl::desc("Disable vector double new-value-stores" )); |
| 76 | |
| 77 | extern cl::opt<bool> ScheduleInlineAsm; |
| 78 | |
| 79 | namespace { |
| 80 | |
| 81 | class HexagonPacketizer : public MachineFunctionPass { |
| 82 | public: |
| 83 | static char ID; |
| 84 | |
| 85 | HexagonPacketizer(bool Min = false) |
| 86 | : MachineFunctionPass(ID), Minimal(Min) {} |
| 87 | |
| 88 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 89 | AU.setPreservesCFG(); |
| 90 | AU.addRequired<AAResultsWrapperPass>(); |
| 91 | AU.addRequired<MachineBranchProbabilityInfoWrapperPass>(); |
| 92 | AU.addRequired<MachineLoopInfoWrapperPass>(); |
| 93 | MachineFunctionPass::getAnalysisUsage(AU); |
| 94 | } |
| 95 | |
| 96 | StringRef getPassName() const override { return "Hexagon Packetizer" ; } |
| 97 | bool runOnMachineFunction(MachineFunction &Fn) override; |
| 98 | |
| 99 | MachineFunctionProperties getRequiredProperties() const override { |
| 100 | return MachineFunctionProperties().setNoVRegs(); |
| 101 | } |
| 102 | |
| 103 | private: |
| 104 | const HexagonInstrInfo *HII = nullptr; |
| 105 | const HexagonRegisterInfo *HRI = nullptr; |
| 106 | const bool Minimal = false; |
| 107 | }; |
| 108 | |
| 109 | } // end anonymous namespace |
| 110 | |
| 111 | char HexagonPacketizer::ID = 0; |
| 112 | |
| 113 | INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer" , |
| 114 | "Hexagon Packetizer" , false, false) |
| 115 | INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass) |
| 116 | INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass) |
| 117 | INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass) |
| 118 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) |
| 119 | INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer" , |
| 120 | "Hexagon Packetizer" , false, false) |
| 121 | |
| 122 | HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF, |
| 123 | MachineLoopInfo &MLI, AAResults *AA, |
| 124 | const MachineBranchProbabilityInfo *MBPI, bool Minimal) |
| 125 | : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI), |
| 126 | Minimal(Minimal) { |
| 127 | HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo(); |
| 128 | HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 129 | |
| 130 | addMutation(Mutation: std::make_unique<HexagonSubtarget::UsrOverflowMutation>()); |
| 131 | addMutation(Mutation: std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>()); |
| 132 | addMutation(Mutation: std::make_unique<HexagonSubtarget::HMXActivateWeightMutation>()); |
| 133 | addMutation(Mutation: std::make_unique<HexagonSubtarget::BankConflictMutation>()); |
| 134 | addMutation(Mutation: std::make_unique<HexagonSubtarget::HLXPseudoMutation>()); |
| 135 | } |
| 136 | |
| 137 | // Check if FirstI modifies a register that SecondI reads. |
| 138 | static bool hasWriteToReadDep(const MachineInstr &FirstI, |
| 139 | const MachineInstr &SecondI, |
| 140 | const TargetRegisterInfo *TRI) { |
| 141 | for (auto &MO : FirstI.operands()) { |
| 142 | if (!MO.isReg() || !MO.isDef()) |
| 143 | continue; |
| 144 | Register R = MO.getReg(); |
| 145 | if (SecondI.readsRegister(Reg: R, TRI)) |
| 146 | return true; |
| 147 | } |
| 148 | return false; |
| 149 | } |
| 150 | |
| 151 | |
| 152 | static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI, |
| 153 | MachineBasicBlock::iterator BundleIt, bool Before) { |
| 154 | MachineBasicBlock::instr_iterator InsertPt; |
| 155 | if (Before) |
| 156 | InsertPt = BundleIt.getInstrIterator(); |
| 157 | else |
| 158 | InsertPt = std::next(x: BundleIt).getInstrIterator(); |
| 159 | |
| 160 | MachineBasicBlock &B = *MI.getParent(); |
| 161 | // The instruction should at least be bundled with the preceding instruction |
| 162 | // (there will always be one, i.e. BUNDLE, if nothing else). |
| 163 | assert(MI.isBundledWithPred()); |
| 164 | if (MI.isBundledWithSucc()) { |
| 165 | MI.clearFlag(Flag: MachineInstr::BundledSucc); |
| 166 | MI.clearFlag(Flag: MachineInstr::BundledPred); |
| 167 | } else { |
| 168 | // If it's not bundled with the successor (i.e. it is the last one |
| 169 | // in the bundle), then we can simply unbundle it from the predecessor, |
| 170 | // which will take care of updating the predecessor's flag. |
| 171 | MI.unbundleFromPred(); |
| 172 | } |
| 173 | B.splice(Where: InsertPt, Other: &B, From: MI.getIterator()); |
| 174 | |
| 175 | // Get the size of the bundle without asserting. |
| 176 | MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator(); |
| 177 | MachineBasicBlock::const_instr_iterator E = B.instr_end(); |
| 178 | unsigned Size = 0; |
| 179 | for (++I; I != E && I->isBundledWithPred(); ++I) |
| 180 | ++Size; |
| 181 | |
| 182 | // If there are still two or more instructions, then there is nothing |
| 183 | // else to be done. |
| 184 | if (Size > 1) |
| 185 | return BundleIt; |
| 186 | |
| 187 | // Otherwise, extract the single instruction out and delete the bundle. |
| 188 | MachineBasicBlock::iterator NextIt = std::next(x: BundleIt); |
| 189 | MachineInstr &SingleI = *BundleIt->getNextNode(); |
| 190 | SingleI.unbundleFromPred(); |
| 191 | assert(!SingleI.isBundledWithSucc()); |
| 192 | BundleIt->eraseFromParent(); |
| 193 | return NextIt; |
| 194 | } |
| 195 | |
| 196 | bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) { |
| 197 | // FIXME: This pass causes verification failures. |
| 198 | MF.getProperties().setFailsVerification(); |
| 199 | |
| 200 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 201 | HII = HST.getInstrInfo(); |
| 202 | HRI = HST.getRegisterInfo(); |
| 203 | auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI(); |
| 204 | auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); |
| 205 | auto *MBPI = |
| 206 | &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); |
| 207 | |
| 208 | if (EnableGenAllInsnClass) |
| 209 | HII->genAllInsnTimingClasses(MF); |
| 210 | |
| 211 | // Instantiate the packetizer. |
| 212 | bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() || |
| 213 | skipFunction(F: MF.getFunction()); |
| 214 | HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI, MinOnly); |
| 215 | |
| 216 | // DFA state table should not be empty. |
| 217 | assert(Packetizer.getResourceTracker() && "Empty DFA table!" ); |
| 218 | |
| 219 | // Loop over all basic blocks and remove KILL pseudo-instructions |
| 220 | // These instructions confuse the dependence analysis. Consider: |
| 221 | // D0 = ... (Insn 0) |
| 222 | // R0 = KILL R0, D0 (Insn 1) |
| 223 | // R0 = ... (Insn 2) |
| 224 | // Here, Insn 1 will result in the dependence graph not emitting an output |
| 225 | // dependence between Insn 0 and Insn 2. This can lead to incorrect |
| 226 | // packetization |
| 227 | for (MachineBasicBlock &MB : MF) { |
| 228 | for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MB)) |
| 229 | if (MI.isKill()) |
| 230 | MB.erase(I: &MI); |
| 231 | } |
| 232 | |
| 233 | // TinyCore with Duplexes: Translate to big-instructions. |
| 234 | if (HST.isTinyCoreWithDuplex()) |
| 235 | HII->translateInstrsForDup(MF, ToBigInstrs: true); |
| 236 | |
| 237 | // Loop over all of the basic blocks. |
| 238 | for (auto &MB : MF) { |
| 239 | auto Begin = MB.begin(), End = MB.end(); |
| 240 | while (Begin != End) { |
| 241 | // Find the first non-boundary starting from the end of the last |
| 242 | // scheduling region. |
| 243 | MachineBasicBlock::iterator RB = Begin; |
| 244 | while (RB != End && HII->isSchedulingBoundary(MI: *RB, MBB: &MB, MF)) |
| 245 | ++RB; |
| 246 | // Find the first boundary starting from the beginning of the new |
| 247 | // region. |
| 248 | MachineBasicBlock::iterator RE = RB; |
| 249 | while (RE != End && !HII->isSchedulingBoundary(MI: *RE, MBB: &MB, MF)) |
| 250 | ++RE; |
| 251 | // Add the scheduling boundary if it's not block end. |
| 252 | if (RE != End) |
| 253 | ++RE; |
| 254 | // If RB == End, then RE == End. |
| 255 | if (RB != End) |
| 256 | Packetizer.PacketizeMIs(MBB: &MB, BeginItr: RB, EndItr: RE); |
| 257 | |
| 258 | Begin = RE; |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | // TinyCore with Duplexes: Translate to tiny-instructions. |
| 263 | if (HST.isTinyCoreWithDuplex()) |
| 264 | HII->translateInstrsForDup(MF, ToBigInstrs: false); |
| 265 | |
| 266 | Packetizer.unpacketizeSoloInstrs(MF); |
| 267 | return true; |
| 268 | } |
| 269 | |
| 270 | // Reserve resources for a constant extender. Trigger an assertion if the |
| 271 | // reservation fails. |
| 272 | void HexagonPacketizerList::reserveResourcesForConstExt() { |
| 273 | if (!tryAllocateResourcesForConstExt(Reserve: true)) |
| 274 | llvm_unreachable("Resources not available" ); |
| 275 | } |
| 276 | |
| 277 | bool HexagonPacketizerList::canReserveResourcesForConstExt() { |
| 278 | return tryAllocateResourcesForConstExt(Reserve: false); |
| 279 | } |
| 280 | |
| 281 | // Allocate resources (i.e. 4 bytes) for constant extender. If succeeded, |
| 282 | // return true, otherwise, return false. |
| 283 | bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) { |
| 284 | auto *ExtMI = MF.CreateMachineInstr(MCID: HII->get(Opcode: Hexagon::A4_ext), DL: DebugLoc()); |
| 285 | bool Avail = ResourceTracker->canReserveResources(MI&: *ExtMI); |
| 286 | if (Reserve && Avail) |
| 287 | ResourceTracker->reserveResources(MI&: *ExtMI); |
| 288 | MF.deleteMachineInstr(MI: ExtMI); |
| 289 | return Avail; |
| 290 | } |
| 291 | |
| 292 | bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI, |
| 293 | SDep::Kind DepType, unsigned DepReg) { |
| 294 | // Check for LR dependence. |
| 295 | if (DepReg == HRI->getRARegister()) |
| 296 | return true; |
| 297 | |
| 298 | if (HII->isDeallocRet(MI)) |
| 299 | if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister()) |
| 300 | return true; |
| 301 | |
| 302 | // Call-like instructions can be packetized with preceding instructions |
| 303 | // that define registers implicitly used or modified by the call. Explicit |
| 304 | // uses are still prohibited, as in the case of indirect calls: |
| 305 | // r0 = ... |
| 306 | // J2_jumpr r0 |
| 307 | if (DepType == SDep::Data) { |
| 308 | for (const MachineOperand &MO : MI.operands()) |
| 309 | if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit()) |
| 310 | return true; |
| 311 | } |
| 312 | |
| 313 | return false; |
| 314 | } |
| 315 | |
| 316 | static bool isRegDependence(const SDep::Kind DepType) { |
| 317 | return DepType == SDep::Data || DepType == SDep::Anti || |
| 318 | DepType == SDep::Output; |
| 319 | } |
| 320 | |
| 321 | static bool isDirectJump(const MachineInstr &MI) { |
| 322 | return MI.getOpcode() == Hexagon::J2_jump; |
| 323 | } |
| 324 | |
| 325 | static bool isSchedBarrier(const MachineInstr &MI) { |
| 326 | switch (MI.getOpcode()) { |
| 327 | case Hexagon::Y2_barrier: |
| 328 | return true; |
| 329 | } |
| 330 | return false; |
| 331 | } |
| 332 | |
| 333 | static bool isControlFlow(const MachineInstr &MI) { |
| 334 | return MI.getDesc().isTerminator() || MI.getDesc().isCall(); |
| 335 | } |
| 336 | |
| 337 | /// Returns true if the instruction modifies a callee-saved register. |
| 338 | static bool doesModifyCalleeSavedReg(const MachineInstr &MI, |
| 339 | const TargetRegisterInfo *TRI) { |
| 340 | const MachineFunction &MF = *MI.getParent()->getParent(); |
| 341 | for (auto *CSR = TRI->getCalleeSavedRegs(MF: &MF); CSR && *CSR; ++CSR) |
| 342 | if (MI.modifiesRegister(Reg: *CSR, TRI)) |
| 343 | return true; |
| 344 | return false; |
| 345 | } |
| 346 | |
| 347 | // Returns true if an instruction can be promoted to .new predicate or |
| 348 | // new-value store. |
| 349 | bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI, |
| 350 | const TargetRegisterClass *NewRC) { |
| 351 | // Vector stores can be predicated, and can be new-value stores, but |
| 352 | // they cannot be predicated on a .new predicate value. |
| 353 | if (NewRC == &Hexagon::PredRegsRegClass) { |
| 354 | if (HII->isHVXVec(MI) && MI.mayStore()) |
| 355 | return false; |
| 356 | return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, MBPI: nullptr) > 0; |
| 357 | } |
| 358 | // If the class is not PredRegs, it could only apply to new-value stores. |
| 359 | return HII->mayBeNewStore(MI); |
| 360 | } |
| 361 | |
| 362 | // Promote an instructiont to its .cur form. |
| 363 | // At this time, we have already made a call to canPromoteToDotCur and made |
| 364 | // sure that it can *indeed* be promoted. |
| 365 | bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI, |
| 366 | SDep::Kind DepType, MachineBasicBlock::iterator &MII, |
| 367 | const TargetRegisterClass* RC) { |
| 368 | assert(DepType == SDep::Data); |
| 369 | int CurOpcode = HII->getDotCurOp(MI); |
| 370 | MI.setDesc(HII->get(Opcode: CurOpcode)); |
| 371 | return true; |
| 372 | } |
| 373 | |
| 374 | void HexagonPacketizerList::cleanUpDotCur() { |
| 375 | MachineInstr *MI = nullptr; |
| 376 | for (auto *BI : CurrentPacketMIs) { |
| 377 | LLVM_DEBUG(dbgs() << "Cleanup packet has " ; BI->dump();); |
| 378 | if (HII->isDotCurInst(MI: *BI)) { |
| 379 | MI = BI; |
| 380 | continue; |
| 381 | } |
| 382 | if (MI) { |
| 383 | for (auto &MO : BI->operands()) |
| 384 | if (MO.isReg() && MO.getReg() == MI->getOperand(i: 0).getReg()) |
| 385 | return; |
| 386 | } |
| 387 | } |
| 388 | if (!MI) |
| 389 | return; |
| 390 | // We did not find a use of the CUR, so de-cur it. |
| 391 | MI->setDesc(HII->get(Opcode: HII->getNonDotCurOp(MI: *MI))); |
| 392 | LLVM_DEBUG(dbgs() << "Demoted CUR " ; MI->dump();); |
| 393 | } |
| 394 | |
| 395 | // Check to see if an instruction can be dot cur. |
| 396 | bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI, |
| 397 | const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, |
| 398 | const TargetRegisterClass *RC) { |
| 399 | if (!HII->isHVXVec(MI)) |
| 400 | return false; |
| 401 | if (!HII->isHVXVec(MI: *MII)) |
| 402 | return false; |
| 403 | |
| 404 | // Already a dot new instruction. |
| 405 | if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI)) |
| 406 | return false; |
| 407 | |
| 408 | if (!HII->mayBeCurLoad(MI)) |
| 409 | return false; |
| 410 | |
| 411 | // The "cur value" cannot come from inline asm. |
| 412 | if (PacketSU->getInstr()->isInlineAsm()) |
| 413 | return false; |
| 414 | |
| 415 | // Make sure candidate instruction uses cur. |
| 416 | LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n" ; MI.dump(); |
| 417 | dbgs() << "in packet\n" ;); |
| 418 | MachineInstr &MJ = *MII; |
| 419 | LLVM_DEBUG({ |
| 420 | dbgs() << "Checking CUR against " ; |
| 421 | MJ.dump(); |
| 422 | }); |
| 423 | Register DestReg = MI.getOperand(i: 0).getReg(); |
| 424 | bool FoundMatch = false; |
| 425 | for (auto &MO : MJ.operands()) |
| 426 | if (MO.isReg() && MO.getReg() == DestReg) |
| 427 | FoundMatch = true; |
| 428 | if (!FoundMatch) |
| 429 | return false; |
| 430 | |
| 431 | // Check for existing uses of a vector register within the packet which |
| 432 | // would be affected by converting a vector load into .cur format. |
| 433 | for (auto *BI : CurrentPacketMIs) { |
| 434 | LLVM_DEBUG(dbgs() << "packet has " ; BI->dump();); |
| 435 | if (BI->readsRegister(Reg: DepReg, TRI: MF.getSubtarget().getRegisterInfo())) |
| 436 | return false; |
| 437 | } |
| 438 | |
| 439 | LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n" ; MI.dump();); |
| 440 | // We can convert the opcode into a .cur. |
| 441 | return true; |
| 442 | } |
| 443 | |
| 444 | // Promote an instruction to its .new form. At this time, we have already |
| 445 | // made a call to canPromoteToDotNew and made sure that it can *indeed* be |
| 446 | // promoted. |
| 447 | bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI, |
| 448 | SDep::Kind DepType, MachineBasicBlock::iterator &MII, |
| 449 | const TargetRegisterClass* RC) { |
| 450 | assert(DepType == SDep::Data); |
| 451 | int NewOpcode; |
| 452 | if (RC == &Hexagon::PredRegsRegClass) |
| 453 | NewOpcode = HII->getDotNewPredOp(MI, MBPI); |
| 454 | else |
| 455 | NewOpcode = HII->getDotNewOp(MI); |
| 456 | MI.setDesc(HII->get(Opcode: NewOpcode)); |
| 457 | return true; |
| 458 | } |
| 459 | |
| 460 | bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) { |
| 461 | int NewOpcode = HII->getDotOldOp(MI); |
| 462 | MI.setDesc(HII->get(Opcode: NewOpcode)); |
| 463 | return true; |
| 464 | } |
| 465 | |
| 466 | bool HexagonPacketizerList::(MachineInstr &MI) { |
| 467 | unsigned Opc = MI.getOpcode(); |
| 468 | switch (Opc) { |
| 469 | case Hexagon::S2_storerd_io: |
| 470 | case Hexagon::S2_storeri_io: |
| 471 | case Hexagon::S2_storerh_io: |
| 472 | case Hexagon::S2_storerb_io: |
| 473 | break; |
| 474 | default: |
| 475 | llvm_unreachable("Unexpected instruction" ); |
| 476 | } |
| 477 | unsigned FrameSize = MF.getFrameInfo().getStackSize(); |
| 478 | MachineOperand &Off = MI.getOperand(i: 1); |
| 479 | int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE); |
| 480 | if (HII->isValidOffset(Opcode: Opc, Offset: NewOff)) { |
| 481 | Off.setImm(NewOff); |
| 482 | return true; |
| 483 | } |
| 484 | return false; |
| 485 | } |
| 486 | |
| 487 | void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) { |
| 488 | unsigned Opc = MI.getOpcode(); |
| 489 | switch (Opc) { |
| 490 | case Hexagon::S2_storerd_io: |
| 491 | case Hexagon::S2_storeri_io: |
| 492 | case Hexagon::S2_storerh_io: |
| 493 | case Hexagon::S2_storerb_io: |
| 494 | break; |
| 495 | default: |
| 496 | llvm_unreachable("Unexpected instruction" ); |
| 497 | } |
| 498 | unsigned FrameSize = MF.getFrameInfo().getStackSize(); |
| 499 | MachineOperand &Off = MI.getOperand(i: 1); |
| 500 | Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE); |
| 501 | } |
| 502 | |
| 503 | /// Return true if we can update the offset in MI so that MI and MJ |
| 504 | /// can be packetized together. |
| 505 | bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) { |
| 506 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 507 | MachineInstr &MI = *SUI->getInstr(); |
| 508 | MachineInstr &MJ = *SUJ->getInstr(); |
| 509 | |
| 510 | unsigned BPI, OPI; |
| 511 | if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BPI, OffsetPos&: OPI)) |
| 512 | return false; |
| 513 | unsigned BPJ, OPJ; |
| 514 | if (!HII->getBaseAndOffsetPosition(MI: MJ, BasePos&: BPJ, OffsetPos&: OPJ)) |
| 515 | return false; |
| 516 | Register Reg = MI.getOperand(i: BPI).getReg(); |
| 517 | if (Reg != MJ.getOperand(i: BPJ).getReg()) |
| 518 | return false; |
| 519 | // Make sure that the dependences do not restrict adding MI to the packet. |
| 520 | // That is, ignore anti dependences, and make sure the only data dependence |
| 521 | // involves the specific register. |
| 522 | for (const auto &PI : SUI->Preds) |
| 523 | if (PI.getKind() != SDep::Anti && |
| 524 | (PI.getKind() != SDep::Data || PI.getReg() != Reg)) |
| 525 | return false; |
| 526 | int Incr; |
| 527 | if (!HII->getIncrementValue(MI: MJ, Value&: Incr)) |
| 528 | return false; |
| 529 | |
| 530 | int64_t Offset = MI.getOperand(i: OPI).getImm(); |
| 531 | if (!HII->isValidOffset(Opcode: MI.getOpcode(), Offset: Offset + Incr)) |
| 532 | return false; |
| 533 | |
| 534 | MI.getOperand(i: OPI).setImm(Offset + Incr); |
| 535 | ChangedOffset = Offset; |
| 536 | return true; |
| 537 | } |
| 538 | |
| 539 | /// Undo the changed offset. This is needed if the instruction cannot be |
| 540 | /// added to the current packet due to a different instruction. |
| 541 | void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) { |
| 542 | unsigned BP, OP; |
| 543 | if (!HII->getBaseAndOffsetPosition(MI, BasePos&: BP, OffsetPos&: OP)) |
| 544 | llvm_unreachable("Unable to find base and offset operands." ); |
| 545 | MI.getOperand(i: OP).setImm(ChangedOffset); |
| 546 | } |
| 547 | |
| 548 | enum PredicateKind { |
| 549 | PK_False, |
| 550 | PK_True, |
| 551 | PK_Unknown |
| 552 | }; |
| 553 | |
| 554 | /// Returns true if an instruction is predicated on p0 and false if it's |
| 555 | /// predicated on !p0. |
| 556 | static PredicateKind getPredicateSense(const MachineInstr &MI, |
| 557 | const HexagonInstrInfo *HII) { |
| 558 | if (!HII->isPredicated(MI)) |
| 559 | return PK_Unknown; |
| 560 | if (HII->isPredicatedTrue(MI)) |
| 561 | return PK_True; |
| 562 | return PK_False; |
| 563 | } |
| 564 | |
| 565 | static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI, |
| 566 | const HexagonInstrInfo *HII) { |
| 567 | assert(HII->isPostIncrement(MI) && "Not a post increment operation." ); |
| 568 | #ifndef NDEBUG |
| 569 | // Post Increment means duplicates. Use dense map to find duplicates in the |
| 570 | // list. Caution: Densemap initializes with the minimum of 64 buckets, |
| 571 | // whereas there are at most 5 operands in the post increment. |
| 572 | DenseSet<unsigned> DefRegsSet; |
| 573 | for (auto &MO : MI.operands()) |
| 574 | if (MO.isReg() && MO.isDef()) |
| 575 | DefRegsSet.insert(MO.getReg()); |
| 576 | |
| 577 | for (auto &MO : MI.operands()) |
| 578 | if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg())) |
| 579 | return MO; |
| 580 | #else |
| 581 | if (MI.mayLoad()) { |
| 582 | const MachineOperand &Op1 = MI.getOperand(i: 1); |
| 583 | // The 2nd operand is always the post increment operand in load. |
| 584 | assert(Op1.isReg() && "Post increment operand has be to a register." ); |
| 585 | return Op1; |
| 586 | } |
| 587 | if (MI.getDesc().mayStore()) { |
| 588 | const MachineOperand &Op0 = MI.getOperand(i: 0); |
| 589 | // The 1st operand is always the post increment operand in store. |
| 590 | assert(Op0.isReg() && "Post increment operand has be to a register." ); |
| 591 | return Op0; |
| 592 | } |
| 593 | #endif |
| 594 | // we should never come here. |
| 595 | llvm_unreachable("mayLoad or mayStore not set for Post Increment operation" ); |
| 596 | } |
| 597 | |
| 598 | // Get the value being stored. |
| 599 | static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) { |
| 600 | // value being stored is always the last operand. |
| 601 | return MI.getOperand(i: MI.getNumOperands()-1); |
| 602 | } |
| 603 | |
| 604 | static bool isLoadAbsSet(const MachineInstr &MI) { |
| 605 | unsigned Opc = MI.getOpcode(); |
| 606 | switch (Opc) { |
| 607 | case Hexagon::L4_loadrd_ap: |
| 608 | case Hexagon::L4_loadrb_ap: |
| 609 | case Hexagon::L4_loadrh_ap: |
| 610 | case Hexagon::L4_loadrub_ap: |
| 611 | case Hexagon::L4_loadruh_ap: |
| 612 | case Hexagon::L4_loadri_ap: |
| 613 | return true; |
| 614 | } |
| 615 | return false; |
| 616 | } |
| 617 | |
| 618 | static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) { |
| 619 | assert(isLoadAbsSet(MI)); |
| 620 | return MI.getOperand(i: 1); |
| 621 | } |
| 622 | |
| 623 | // Can be new value store? |
| 624 | // Following restrictions are to be respected in convert a store into |
| 625 | // a new value store. |
| 626 | // 1. If an instruction uses auto-increment, its address register cannot |
| 627 | // be a new-value register. Arch Spec 5.4.2.1 |
| 628 | // 2. If an instruction uses absolute-set addressing mode, its address |
| 629 | // register cannot be a new-value register. Arch Spec 5.4.2.1. |
| 630 | // 3. If an instruction produces a 64-bit result, its registers cannot be used |
| 631 | // as new-value registers. Arch Spec 5.4.2.2. |
| 632 | // 4. If the instruction that sets the new-value register is conditional, then |
| 633 | // the instruction that uses the new-value register must also be conditional, |
| 634 | // and both must always have their predicates evaluate identically. |
| 635 | // Arch Spec 5.4.2.3. |
| 636 | // 5. There is an implied restriction that a packet cannot have another store, |
| 637 | // if there is a new value store in the packet. Corollary: if there is |
| 638 | // already a store in a packet, there can not be a new value store. |
| 639 | // Arch Spec: 3.4.4.2 |
| 640 | bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI, |
| 641 | const MachineInstr &PacketMI, unsigned DepReg) { |
| 642 | // Make sure we are looking at the store, that can be promoted. |
| 643 | if (!HII->mayBeNewStore(MI)) |
| 644 | return false; |
| 645 | |
| 646 | // Make sure there is dependency and can be new value'd. |
| 647 | const MachineOperand &Val = getStoreValueOperand(MI); |
| 648 | if (Val.isReg() && Val.getReg() != DepReg) |
| 649 | return false; |
| 650 | |
| 651 | const MCInstrDesc& MCID = PacketMI.getDesc(); |
| 652 | |
| 653 | // First operand is always the result. |
| 654 | const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, OpNum: 0); |
| 655 | // Double regs can not feed into new value store: PRM section: 5.4.2.2. |
| 656 | if (PacketRC == &Hexagon::DoubleRegsRegClass) |
| 657 | return false; |
| 658 | |
| 659 | // New-value stores are of class NV (slot 0), dual stores require class ST |
| 660 | // in slot 0 (PRM 5.5). |
| 661 | for (auto *I : CurrentPacketMIs) { |
| 662 | SUnit *PacketSU = MIToSUnit.find(x: I)->second; |
| 663 | if (PacketSU->getInstr()->mayStore()) |
| 664 | return false; |
| 665 | } |
| 666 | |
| 667 | // Make sure it's NOT the post increment register that we are going to |
| 668 | // new value. |
| 669 | if (HII->isPostIncrement(MI) && |
| 670 | getPostIncrementOperand(MI, HII).getReg() == DepReg) { |
| 671 | return false; |
| 672 | } |
| 673 | |
| 674 | if (HII->isPostIncrement(MI: PacketMI) && PacketMI.mayLoad() && |
| 675 | getPostIncrementOperand(MI: PacketMI, HII).getReg() == DepReg) { |
| 676 | // If source is post_inc, or absolute-set addressing, it can not feed |
| 677 | // into new value store |
| 678 | // r3 = memw(r2++#4) |
| 679 | // memw(r30 + #-1404) = r2.new -> can not be new value store |
| 680 | // arch spec section: 5.4.2.1. |
| 681 | return false; |
| 682 | } |
| 683 | |
| 684 | if (isLoadAbsSet(MI: PacketMI) && getAbsSetOperand(MI: PacketMI).getReg() == DepReg) |
| 685 | return false; |
| 686 | |
| 687 | // If the source that feeds the store is predicated, new value store must |
| 688 | // also be predicated. |
| 689 | if (HII->isPredicated(MI: PacketMI)) { |
| 690 | if (!HII->isPredicated(MI)) |
| 691 | return false; |
| 692 | |
| 693 | // Check to make sure that they both will have their predicates |
| 694 | // evaluate identically. |
| 695 | unsigned predRegNumSrc = 0; |
| 696 | unsigned predRegNumDst = 0; |
| 697 | const TargetRegisterClass* predRegClass = nullptr; |
| 698 | |
| 699 | // Get predicate register used in the source instruction. |
| 700 | for (auto &MO : PacketMI.operands()) { |
| 701 | if (!MO.isReg()) |
| 702 | continue; |
| 703 | predRegNumSrc = MO.getReg(); |
| 704 | predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumSrc); |
| 705 | if (predRegClass == &Hexagon::PredRegsRegClass) |
| 706 | break; |
| 707 | } |
| 708 | assert((predRegClass == &Hexagon::PredRegsRegClass) && |
| 709 | "predicate register not found in a predicated PacketMI instruction" ); |
| 710 | |
| 711 | // Get predicate register used in new-value store instruction. |
| 712 | for (auto &MO : MI.operands()) { |
| 713 | if (!MO.isReg()) |
| 714 | continue; |
| 715 | predRegNumDst = MO.getReg(); |
| 716 | predRegClass = HRI->getMinimalPhysRegClass(Reg: predRegNumDst); |
| 717 | if (predRegClass == &Hexagon::PredRegsRegClass) |
| 718 | break; |
| 719 | } |
| 720 | assert((predRegClass == &Hexagon::PredRegsRegClass) && |
| 721 | "predicate register not found in a predicated MI instruction" ); |
| 722 | |
| 723 | // New-value register producer and user (store) need to satisfy these |
| 724 | // constraints: |
| 725 | // 1) Both instructions should be predicated on the same register. |
| 726 | // 2) If producer of the new-value register is .new predicated then store |
| 727 | // should also be .new predicated and if producer is not .new predicated |
| 728 | // then store should not be .new predicated. |
| 729 | // 3) Both new-value register producer and user should have same predicate |
| 730 | // sense, i.e, either both should be negated or both should be non-negated. |
| 731 | if (predRegNumDst != predRegNumSrc || |
| 732 | HII->isDotNewInst(MI: PacketMI) != HII->isDotNewInst(MI) || |
| 733 | getPredicateSense(MI, HII) != getPredicateSense(MI: PacketMI, HII)) |
| 734 | return false; |
| 735 | } |
| 736 | |
| 737 | // Make sure that other than the new-value register no other store instruction |
| 738 | // register has been modified in the same packet. Predicate registers can be |
| 739 | // modified by they should not be modified between the producer and the store |
| 740 | // instruction as it will make them both conditional on different values. |
| 741 | // We already know this to be true for all the instructions before and |
| 742 | // including PacketMI. However, we need to perform the check for the |
| 743 | // remaining instructions in the packet. |
| 744 | |
| 745 | unsigned StartCheck = 0; |
| 746 | |
| 747 | for (auto *I : CurrentPacketMIs) { |
| 748 | SUnit *TempSU = MIToSUnit.find(x: I)->second; |
| 749 | MachineInstr &TempMI = *TempSU->getInstr(); |
| 750 | |
| 751 | // Following condition is true for all the instructions until PacketMI is |
| 752 | // reached (StartCheck is set to 0 before the for loop). |
| 753 | // StartCheck flag is 1 for all the instructions after PacketMI. |
| 754 | if (&TempMI != &PacketMI && !StartCheck) // Start processing only after |
| 755 | continue; // encountering PacketMI. |
| 756 | |
| 757 | StartCheck = 1; |
| 758 | if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence. |
| 759 | continue; |
| 760 | |
| 761 | for (auto &MO : MI.operands()) |
| 762 | if (MO.isReg() && TempSU->getInstr()->modifiesRegister(Reg: MO.getReg(), TRI: HRI)) |
| 763 | return false; |
| 764 | } |
| 765 | |
| 766 | // Make sure that for non-POST_INC stores: |
| 767 | // 1. The only use of reg is DepReg and no other registers. |
| 768 | // This handles base+index registers. |
| 769 | // The following store can not be dot new. |
| 770 | // Eg. r0 = add(r0, #3) |
| 771 | // memw(r1+r0<<#2) = r0 |
| 772 | if (!HII->isPostIncrement(MI)) { |
| 773 | for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) { |
| 774 | const MachineOperand &MO = MI.getOperand(i: opNum); |
| 775 | if (MO.isReg() && MO.getReg() == DepReg) |
| 776 | return false; |
| 777 | } |
| 778 | } |
| 779 | |
| 780 | // If data definition is because of implicit definition of the register, |
| 781 | // do not newify the store. Eg. |
| 782 | // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12 |
| 783 | // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343] |
| 784 | for (auto &MO : PacketMI.operands()) { |
| 785 | if (MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg)) |
| 786 | return false; |
| 787 | if (!MO.isReg() || !MO.isDef() || !MO.isImplicit()) |
| 788 | continue; |
| 789 | Register R = MO.getReg(); |
| 790 | if (R == DepReg || HRI->isSuperRegister(RegA: DepReg, RegB: R)) |
| 791 | return false; |
| 792 | } |
| 793 | |
| 794 | // Handle imp-use of super reg case. There is a target independent side |
| 795 | // change that should prevent this situation but I am handling it for |
| 796 | // just-in-case. For example, we cannot newify R2 in the following case: |
| 797 | // %r3 = A2_tfrsi 0; |
| 798 | // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1; |
| 799 | for (auto &MO : MI.operands()) { |
| 800 | if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg) |
| 801 | return false; |
| 802 | } |
| 803 | |
| 804 | // Can be dot new store. |
| 805 | return true; |
| 806 | } |
| 807 | |
| 808 | // Can this MI to promoted to either new value store or new value jump. |
| 809 | bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI, |
| 810 | const SUnit *PacketSU, unsigned DepReg, |
| 811 | MachineBasicBlock::iterator &MII) { |
| 812 | if (!HII->mayBeNewStore(MI)) |
| 813 | return false; |
| 814 | |
| 815 | // Check to see the store can be new value'ed. |
| 816 | MachineInstr &PacketMI = *PacketSU->getInstr(); |
| 817 | if (canPromoteToNewValueStore(MI, PacketMI, DepReg)) |
| 818 | return true; |
| 819 | |
| 820 | // Check to see the compare/jump can be new value'ed. |
| 821 | // This is done as a pass on its own. Don't need to check it here. |
| 822 | return false; |
| 823 | } |
| 824 | |
| 825 | static bool isImplicitDependency(const MachineInstr &I, bool CheckDef, |
| 826 | unsigned DepReg) { |
| 827 | for (auto &MO : I.operands()) { |
| 828 | if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(PhysReg: DepReg)) |
| 829 | return true; |
| 830 | if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit()) |
| 831 | continue; |
| 832 | if (CheckDef == MO.isDef()) |
| 833 | return true; |
| 834 | } |
| 835 | return false; |
| 836 | } |
| 837 | |
| 838 | // Check to see if an instruction can be dot new. |
| 839 | bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI, |
| 840 | const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, |
| 841 | const TargetRegisterClass* RC) { |
| 842 | // Already a dot new instruction. |
| 843 | if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI)) |
| 844 | return false; |
| 845 | |
| 846 | if (!isNewifiable(MI, NewRC: RC)) |
| 847 | return false; |
| 848 | |
| 849 | const MachineInstr &PI = *PacketSU->getInstr(); |
| 850 | |
| 851 | // The "new value" cannot come from inline asm. |
| 852 | if (PI.isInlineAsm()) |
| 853 | return false; |
| 854 | |
| 855 | // IMPLICIT_DEFs and FREEZEs won't materialize as real instructions, so .new |
| 856 | // makes no sense. |
| 857 | if (PI.isImplicitDef() || PI.getOpcode() == TargetOpcode::FREEZE) |
| 858 | return false; |
| 859 | |
| 860 | // If dependency is through an implicitly defined register, we should not |
| 861 | // newify the use. |
| 862 | if (isImplicitDependency(I: PI, CheckDef: true, DepReg) || |
| 863 | isImplicitDependency(I: MI, CheckDef: false, DepReg)) |
| 864 | return false; |
| 865 | |
| 866 | const MCInstrDesc& MCID = PI.getDesc(); |
| 867 | const TargetRegisterClass *VecRC = HII->getRegClass(MCID, OpNum: 0); |
| 868 | if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass) |
| 869 | return false; |
| 870 | |
| 871 | // predicate .new |
| 872 | if (RC == &Hexagon::PredRegsRegClass) |
| 873 | return HII->predCanBeUsedAsDotNew(MI: PI, PredReg: DepReg); |
| 874 | |
| 875 | if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI)) |
| 876 | return false; |
| 877 | |
| 878 | // Create a dot new machine instruction to see if resources can be |
| 879 | // allocated. If not, bail out now. |
| 880 | int NewOpcode = (RC != &Hexagon::PredRegsRegClass) ? HII->getDotNewOp(MI) : |
| 881 | HII->getDotNewPredOp(MI, MBPI); |
| 882 | const MCInstrDesc &D = HII->get(Opcode: NewOpcode); |
| 883 | MachineInstr *NewMI = MF.CreateMachineInstr(MCID: D, DL: DebugLoc()); |
| 884 | bool ResourcesAvailable = ResourceTracker->canReserveResources(MI&: *NewMI); |
| 885 | MF.deleteMachineInstr(MI: NewMI); |
| 886 | if (!ResourcesAvailable) |
| 887 | return false; |
| 888 | |
| 889 | // New Value Store only. New Value Jump generated as a separate pass. |
| 890 | if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII)) |
| 891 | return false; |
| 892 | |
| 893 | return true; |
| 894 | } |
| 895 | |
| 896 | // Go through the packet instructions and search for an anti dependency between |
| 897 | // them and DepReg from MI. Consider this case: |
| 898 | // Trying to add |
| 899 | // a) %r1 = TFRI_cdNotPt %p3, 2 |
| 900 | // to this packet: |
| 901 | // { |
| 902 | // b) %p0 = C2_or killed %p3, killed %p0 |
| 903 | // c) %p3 = C2_tfrrp %r23 |
| 904 | // d) %r1 = C2_cmovenewit %p3, 4 |
| 905 | // } |
| 906 | // The P3 from a) and d) will be complements after |
| 907 | // a)'s P3 is converted to .new form |
| 908 | // Anti-dep between c) and b) is irrelevant for this case |
| 909 | bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI, |
| 910 | unsigned DepReg) { |
| 911 | SUnit *PacketSUDep = MIToSUnit.find(x: &MI)->second; |
| 912 | |
| 913 | for (auto *I : CurrentPacketMIs) { |
| 914 | // We only care for dependencies to predicated instructions |
| 915 | if (!HII->isPredicated(MI: *I)) |
| 916 | continue; |
| 917 | |
| 918 | // Scheduling Unit for current insn in the packet |
| 919 | SUnit *PacketSU = MIToSUnit.find(x: I)->second; |
| 920 | |
| 921 | // Look at dependencies between current members of the packet and |
| 922 | // predicate defining instruction MI. Make sure that dependency is |
| 923 | // on the exact register we care about. |
| 924 | if (PacketSU->isSucc(N: PacketSUDep)) { |
| 925 | for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { |
| 926 | auto &Dep = PacketSU->Succs[i]; |
| 927 | if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti && |
| 928 | Dep.getReg() == DepReg) |
| 929 | return true; |
| 930 | } |
| 931 | } |
| 932 | } |
| 933 | |
| 934 | return false; |
| 935 | } |
| 936 | |
| 937 | /// Gets the predicate register of a predicated instruction. |
| 938 | static unsigned getPredicatedRegister(MachineInstr &MI, |
| 939 | const HexagonInstrInfo *QII) { |
| 940 | /// We use the following rule: The first predicate register that is a use is |
| 941 | /// the predicate register of a predicated instruction. |
| 942 | assert(QII->isPredicated(MI) && "Must be predicated instruction" ); |
| 943 | |
| 944 | for (auto &Op : MI.operands()) { |
| 945 | if (Op.isReg() && Op.getReg() && Op.isUse() && |
| 946 | Hexagon::PredRegsRegClass.contains(Reg: Op.getReg())) |
| 947 | return Op.getReg(); |
| 948 | } |
| 949 | |
| 950 | llvm_unreachable("Unknown instruction operand layout" ); |
| 951 | return 0; |
| 952 | } |
| 953 | |
| 954 | // Given two predicated instructions, this function detects whether |
| 955 | // the predicates are complements. |
| 956 | bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1, |
| 957 | MachineInstr &MI2) { |
| 958 | // If we don't know the predicate sense of the instructions bail out early, we |
| 959 | // need it later. |
| 960 | if (getPredicateSense(MI: MI1, HII) == PK_Unknown || |
| 961 | getPredicateSense(MI: MI2, HII) == PK_Unknown) |
| 962 | return false; |
| 963 | |
| 964 | // Scheduling unit for candidate. |
| 965 | SUnit *SU = MIToSUnit[&MI1]; |
| 966 | |
| 967 | // One corner case deals with the following scenario: |
| 968 | // Trying to add |
| 969 | // a) %r24 = A2_tfrt %p0, %r25 |
| 970 | // to this packet: |
| 971 | // { |
| 972 | // b) %r25 = A2_tfrf %p0, %r24 |
| 973 | // c) %p0 = C2_cmpeqi %r26, 1 |
| 974 | // } |
| 975 | // |
| 976 | // On general check a) and b) are complements, but presence of c) will |
| 977 | // convert a) to .new form, and then it is not a complement. |
| 978 | // We attempt to detect it by analyzing existing dependencies in the packet. |
| 979 | |
| 980 | // Analyze relationships between all existing members of the packet. |
| 981 | // Look for Anti dependency on the same predicate reg as used in the |
| 982 | // candidate. |
| 983 | for (auto *I : CurrentPacketMIs) { |
| 984 | // Scheduling Unit for current insn in the packet. |
| 985 | SUnit *PacketSU = MIToSUnit.find(x: I)->second; |
| 986 | |
| 987 | // If this instruction in the packet is succeeded by the candidate... |
| 988 | if (PacketSU->isSucc(N: SU)) { |
| 989 | for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { |
| 990 | auto Dep = PacketSU->Succs[i]; |
| 991 | // The corner case exist when there is true data dependency between |
| 992 | // candidate and one of current packet members, this dep is on |
| 993 | // predicate reg, and there already exist anti dep on the same pred in |
| 994 | // the packet. |
| 995 | if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data && |
| 996 | Hexagon::PredRegsRegClass.contains(Reg: Dep.getReg())) { |
| 997 | // Here I know that I is predicate setting instruction with true |
| 998 | // data dep to candidate on the register we care about - c) in the |
| 999 | // above example. Now I need to see if there is an anti dependency |
| 1000 | // from c) to any other instruction in the same packet on the pred |
| 1001 | // reg of interest. |
| 1002 | if (restrictingDepExistInPacket(MI&: *I, DepReg: Dep.getReg())) |
| 1003 | return false; |
| 1004 | } |
| 1005 | } |
| 1006 | } |
| 1007 | } |
| 1008 | |
| 1009 | // If the above case does not apply, check regular complement condition. |
| 1010 | // Check that the predicate register is the same and that the predicate |
| 1011 | // sense is different We also need to differentiate .old vs. .new: !p0 |
| 1012 | // is not complementary to p0.new. |
| 1013 | unsigned PReg1 = getPredicatedRegister(MI&: MI1, QII: HII); |
| 1014 | unsigned PReg2 = getPredicatedRegister(MI&: MI2, QII: HII); |
| 1015 | return PReg1 == PReg2 && |
| 1016 | Hexagon::PredRegsRegClass.contains(Reg: PReg1) && |
| 1017 | Hexagon::PredRegsRegClass.contains(Reg: PReg2) && |
| 1018 | getPredicateSense(MI: MI1, HII) != getPredicateSense(MI: MI2, HII) && |
| 1019 | HII->isDotNewInst(MI: MI1) == HII->isDotNewInst(MI: MI2); |
| 1020 | } |
| 1021 | |
| 1022 | // Initialize packetizer flags. |
| 1023 | void HexagonPacketizerList::initPacketizerState() { |
| 1024 | Dependence = false; |
| 1025 | PromotedToDotNew = false; |
| 1026 | GlueToNewValueJump = false; |
| 1027 | GlueAllocframeStore = false; |
| 1028 | FoundSequentialDependence = false; |
| 1029 | ChangedOffset = INT64_MAX; |
| 1030 | } |
| 1031 | |
| 1032 | // Ignore bundling of pseudo instructions. |
| 1033 | bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI, |
| 1034 | const MachineBasicBlock *) { |
| 1035 | if (MI.isDebugInstr()) |
| 1036 | return true; |
| 1037 | |
| 1038 | if (MI.isCFIInstruction()) |
| 1039 | return false; |
| 1040 | |
| 1041 | // We must print out inline assembly. |
| 1042 | if (MI.isInlineAsm()) |
| 1043 | return false; |
| 1044 | |
| 1045 | if (MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::FREEZE) |
| 1046 | return false; |
| 1047 | |
| 1048 | // We check if MI has any functional units mapped to it. If it doesn't, |
| 1049 | // we ignore the instruction. |
| 1050 | const MCInstrDesc& TID = MI.getDesc(); |
| 1051 | auto *IS = ResourceTracker->getInstrItins()->beginStage(ItinClassIndx: TID.getSchedClass()); |
| 1052 | return !IS->getUnits(); |
| 1053 | } |
| 1054 | |
| 1055 | bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) { |
| 1056 | // Ensure any bundles created by gather packetize remain separate. |
| 1057 | if (MI.isBundle()) |
| 1058 | return true; |
| 1059 | |
| 1060 | if (MI.isEHLabel() || MI.isCFIInstruction()) |
| 1061 | return true; |
| 1062 | |
| 1063 | // Consider inline asm to not be a solo instruction by default. |
| 1064 | // Inline asm will be put in a packet temporarily, but then it will be |
| 1065 | // removed, and placed outside of the packet (before or after, depending |
| 1066 | // on dependencies). This is to reduce the impact of inline asm as a |
| 1067 | // "packet splitting" instruction. |
| 1068 | if (MI.isInlineAsm() && !ScheduleInlineAsm) |
| 1069 | return true; |
| 1070 | |
| 1071 | if (isSchedBarrier(MI)) |
| 1072 | return true; |
| 1073 | |
| 1074 | if (HII->isSolo(MI)) |
| 1075 | return true; |
| 1076 | |
| 1077 | if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER || |
| 1078 | MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT || |
| 1079 | MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL || |
| 1080 | MI.getOpcode() == Hexagon::PATCHABLE_EVENT_CALL || |
| 1081 | MI.getOpcode() == Hexagon::PATCHABLE_TYPED_EVENT_CALL) |
| 1082 | return true; |
| 1083 | |
| 1084 | if (MI.getOpcode() == Hexagon::A2_nop) |
| 1085 | return true; |
| 1086 | |
| 1087 | return false; |
| 1088 | } |
| 1089 | |
| 1090 | // Quick check if instructions MI and MJ cannot coexist in the same packet. |
| 1091 | // Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm", |
| 1092 | // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm". |
| 1093 | // For full test call this function twice: |
| 1094 | // cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI) |
| 1095 | // Doing the test only one way saves the amount of code in this function, |
| 1096 | // since every test would need to be repeated with the MI and MJ reversed. |
| 1097 | static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ, |
| 1098 | const HexagonInstrInfo &HII) { |
| 1099 | const MachineFunction *MF = MI.getParent()->getParent(); |
| 1100 | if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() && |
| 1101 | HII.isHVXMemWithAIndirect(I: MI, J: MJ)) |
| 1102 | return true; |
| 1103 | |
| 1104 | // Don't allow a store and an instruction that must be in slot0 and |
| 1105 | // doesn't allow a slot1 instruction. |
| 1106 | if (MI.mayStore() && HII.isRestrictNoSlot1Store(MI: MJ) && HII.isPureSlot0(MI: MJ)) |
| 1107 | return true; |
| 1108 | |
| 1109 | // An inline asm cannot be together with a branch, because we may not be |
| 1110 | // able to remove the asm out after packetizing (i.e. if the asm must be |
| 1111 | // moved past the bundle). Similarly, two asms cannot be together to avoid |
| 1112 | // complications when determining their relative order outside of a bundle. |
| 1113 | if (MI.isInlineAsm()) |
| 1114 | return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() || |
| 1115 | MJ.isCall() || MJ.isTerminator(); |
| 1116 | |
| 1117 | // New-value stores cannot coexist with any other stores. |
| 1118 | if (HII.isNewValueStore(MI) && MJ.mayStore()) |
| 1119 | return true; |
| 1120 | |
| 1121 | // An HLX instruction with a 64-bit scalar source cannot be paired with |
| 1122 | // another HLX instruction using a scalar or immediate source. Loads and |
| 1123 | // stores are exempt because they do not use the shared scalar resource. |
| 1124 | if (HII.isHLXVec(MI) && HII.isHLXVec(MI: MJ) && !MJ.mayLoad() && !MJ.mayStore()) { |
| 1125 | auto getScalarUses = [](const MachineInstr &MI) { |
| 1126 | unsigned Uses = 0; |
| 1127 | for (const MachineOperand &MO : MI.explicit_uses()) { |
| 1128 | if (MO.isImm()) |
| 1129 | Uses |= 1; |
| 1130 | else if (MO.isReg() && Hexagon::IntRegsRegClass.contains(Reg: MO.getReg())) |
| 1131 | Uses |= 2; |
| 1132 | else if (MO.isReg() && |
| 1133 | Hexagon::DoubleRegsRegClass.contains(Reg: MO.getReg())) |
| 1134 | Uses |= 4; |
| 1135 | } |
| 1136 | return Uses; |
| 1137 | }; |
| 1138 | |
| 1139 | if ((getScalarUses(MI) & 4) && getScalarUses(MJ)) |
| 1140 | return true; |
| 1141 | } |
| 1142 | |
| 1143 | switch (MI.getOpcode()) { |
| 1144 | case Hexagon::S2_storew_locked: |
| 1145 | case Hexagon::S4_stored_locked: |
| 1146 | case Hexagon::L2_loadw_locked: |
| 1147 | case Hexagon::L4_loadd_locked: |
| 1148 | case Hexagon::Y2_dccleana: |
| 1149 | case Hexagon::Y2_dccleaninva: |
| 1150 | case Hexagon::Y2_dcinva: |
| 1151 | case Hexagon::Y2_dczeroa: |
| 1152 | case Hexagon::Y4_l2fetch: |
| 1153 | case Hexagon::Y5_l2fetch: { |
| 1154 | // These instructions can only be grouped with ALU32 or non-floating-point |
| 1155 | // XTYPE instructions. Since there is no convenient way of identifying fp |
| 1156 | // XTYPE instructions, only allow grouping with ALU32 for now. |
| 1157 | unsigned TJ = HII.getType(MI: MJ); |
| 1158 | if (TJ != HexagonII::TypeALU32_2op && |
| 1159 | TJ != HexagonII::TypeALU32_3op && |
| 1160 | TJ != HexagonII::TypeALU32_ADDI) |
| 1161 | return true; |
| 1162 | break; |
| 1163 | } |
| 1164 | case Hexagon::LX_S16b_pseudo: { |
| 1165 | // The store pseudo expands to a copy of the stored register followed by a |
| 1166 | // new-value store. Do not packetize it with an instruction defining that |
| 1167 | // register, otherwise the packet would have two definitions of it. |
| 1168 | Register StReg = MI.getOperand(i: 1).getReg(); |
| 1169 | const HexagonRegisterInfo &HRI = HII.getRegisterInfo(); |
| 1170 | if (any_of(Range: MJ.all_defs(), P: [&](const MachineOperand &Def) { |
| 1171 | return HRI.isSuperRegisterEq(RegA: StReg, RegB: Def.getReg()); |
| 1172 | })) |
| 1173 | return true; |
| 1174 | break; |
| 1175 | } |
| 1176 | default: |
| 1177 | break; |
| 1178 | } |
| 1179 | |
| 1180 | // "False" really means that the quick check failed to determine if |
| 1181 | // I and J cannot coexist. |
| 1182 | return false; |
| 1183 | } |
| 1184 | |
| 1185 | // Full, symmetric check. |
| 1186 | bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI, |
| 1187 | const MachineInstr &MJ) { |
| 1188 | return cannotCoexistAsymm(MI, MJ, HII: *HII) || cannotCoexistAsymm(MI: MJ, MJ: MI, HII: *HII); |
| 1189 | } |
| 1190 | |
| 1191 | void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) { |
| 1192 | for (auto &B : MF) { |
| 1193 | MachineBasicBlock::iterator BundleIt; |
| 1194 | for (MachineInstr &MI : llvm::make_early_inc_range(Range: B.instrs())) { |
| 1195 | if (MI.isBundle()) |
| 1196 | BundleIt = MI.getIterator(); |
| 1197 | if (!MI.isInsideBundle()) |
| 1198 | continue; |
| 1199 | |
| 1200 | // Decide on where to insert the instruction that we are pulling out. |
| 1201 | // Debug instructions always go before the bundle, but the placement of |
| 1202 | // INLINE_ASM depends on potential dependencies. By default, try to |
| 1203 | // put it before the bundle, but if the asm writes to a register that |
| 1204 | // other instructions in the bundle read, then we need to place it |
| 1205 | // after the bundle (to preserve the bundle semantics). |
| 1206 | bool InsertBeforeBundle; |
| 1207 | if (MI.isInlineAsm()) |
| 1208 | InsertBeforeBundle = !hasWriteToReadDep(FirstI: MI, SecondI: *BundleIt, TRI: HRI); |
| 1209 | else if (MI.isDebugInstr()) |
| 1210 | InsertBeforeBundle = true; |
| 1211 | else |
| 1212 | continue; |
| 1213 | |
| 1214 | BundleIt = moveInstrOut(MI, BundleIt, Before: InsertBeforeBundle); |
| 1215 | } |
| 1216 | } |
| 1217 | } |
| 1218 | |
| 1219 | // Check if a given instruction is of class "system". |
| 1220 | static bool isSystemInstr(const MachineInstr &MI) { |
| 1221 | unsigned Opc = MI.getOpcode(); |
| 1222 | switch (Opc) { |
| 1223 | case Hexagon::Y2_barrier: |
| 1224 | case Hexagon::Y2_dcfetchbo: |
| 1225 | case Hexagon::Y4_l2fetch: |
| 1226 | case Hexagon::Y5_l2fetch: |
| 1227 | return true; |
| 1228 | } |
| 1229 | return false; |
| 1230 | } |
| 1231 | |
| 1232 | bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I, |
| 1233 | const MachineInstr &J) { |
| 1234 | // The dependence graph may not include edges between dead definitions, |
| 1235 | // so without extra checks, we could end up packetizing two instruction |
| 1236 | // defining the same (dead) register. |
| 1237 | if (I.isCall() || J.isCall()) |
| 1238 | return false; |
| 1239 | if (HII->isPredicated(MI: I) || HII->isPredicated(MI: J)) |
| 1240 | return false; |
| 1241 | |
| 1242 | BitVector DeadDefs(Hexagon::NUM_TARGET_REGS); |
| 1243 | for (auto &MO : I.operands()) { |
| 1244 | if (!MO.isReg() || !MO.isDef() || !MO.isDead()) |
| 1245 | continue; |
| 1246 | DeadDefs[MO.getReg()] = true; |
| 1247 | } |
| 1248 | |
| 1249 | for (auto &MO : J.operands()) { |
| 1250 | if (!MO.isReg() || !MO.isDef() || !MO.isDead()) |
| 1251 | continue; |
| 1252 | Register R = MO.getReg(); |
| 1253 | if (R != Hexagon::USR_OVF && DeadDefs[R]) |
| 1254 | return true; |
| 1255 | } |
| 1256 | return false; |
| 1257 | } |
| 1258 | |
| 1259 | bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I, |
| 1260 | const MachineInstr &J) { |
| 1261 | // A save callee-save register function call can only be in a packet |
| 1262 | // with instructions that don't write to the callee-save registers. |
| 1263 | if ((HII->isSaveCalleeSavedRegsCall(MI: I) && |
| 1264 | doesModifyCalleeSavedReg(MI: J, TRI: HRI)) || |
| 1265 | (HII->isSaveCalleeSavedRegsCall(MI: J) && |
| 1266 | doesModifyCalleeSavedReg(MI: I, TRI: HRI))) |
| 1267 | return true; |
| 1268 | |
| 1269 | // Two control flow instructions cannot go in the same packet. |
| 1270 | if (isControlFlow(MI: I) && isControlFlow(MI: J)) |
| 1271 | return true; |
| 1272 | |
| 1273 | // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot |
| 1274 | // contain a speculative indirect jump, |
| 1275 | // a new-value compare jump or a dealloc_return. |
| 1276 | auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool { |
| 1277 | if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI)) |
| 1278 | return true; |
| 1279 | if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI)) |
| 1280 | return true; |
| 1281 | return false; |
| 1282 | }; |
| 1283 | |
| 1284 | if (HII->isLoopN(MI: I) && isBadForLoopN(J)) |
| 1285 | return true; |
| 1286 | if (HII->isLoopN(MI: J) && isBadForLoopN(I)) |
| 1287 | return true; |
| 1288 | |
| 1289 | // dealloc_return cannot appear in the same packet as a conditional or |
| 1290 | // unconditional jump. |
| 1291 | return HII->isDeallocRet(MI: I) && |
| 1292 | (J.isBranch() || J.isCall() || J.isBarrier()); |
| 1293 | } |
| 1294 | |
| 1295 | bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I, |
| 1296 | const MachineInstr &J) { |
| 1297 | // Adding I to a packet that has J. |
| 1298 | |
| 1299 | // Regmasks are not reflected in the scheduling dependency graph, so |
| 1300 | // we need to check them manually. This code assumes that regmasks only |
| 1301 | // occur on calls, and the problematic case is when we add an instruction |
| 1302 | // defining a register R to a packet that has a call that clobbers R via |
| 1303 | // a regmask. Those cannot be packetized together, because the call will |
| 1304 | // be executed last. That's also a reason why it is ok to add a call |
| 1305 | // clobbering R to a packet that defines R. |
| 1306 | |
| 1307 | // Look for regmasks in J. |
| 1308 | for (const MachineOperand &OpJ : J.operands()) { |
| 1309 | if (!OpJ.isRegMask()) |
| 1310 | continue; |
| 1311 | assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call" ); |
| 1312 | for (const MachineOperand &OpI : I.operands()) { |
| 1313 | if (OpI.isReg()) { |
| 1314 | if (OpJ.clobbersPhysReg(PhysReg: OpI.getReg())) |
| 1315 | return true; |
| 1316 | } else if (OpI.isRegMask()) { |
| 1317 | // Both are regmasks. Assume that they intersect. |
| 1318 | return true; |
| 1319 | } |
| 1320 | } |
| 1321 | } |
| 1322 | return false; |
| 1323 | } |
| 1324 | |
| 1325 | bool HexagonPacketizerList::hasDualStoreDependence(const MachineInstr &I, |
| 1326 | const MachineInstr &J) { |
| 1327 | bool SysI = isSystemInstr(MI: I), SysJ = isSystemInstr(MI: J); |
| 1328 | bool StoreI = I.mayStore(), StoreJ = J.mayStore(); |
| 1329 | if ((SysI && StoreJ) || (SysJ && StoreI)) |
| 1330 | return true; |
| 1331 | |
| 1332 | if (StoreI && StoreJ) { |
| 1333 | if (HII->isNewValueInst(MI: J) || HII->isMemOp(MI: J) || HII->isMemOp(MI: I)) |
| 1334 | return true; |
| 1335 | } else { |
| 1336 | // A memop cannot be in the same packet with another memop or a store. |
| 1337 | // Two stores can be together, but here I and J cannot both be stores. |
| 1338 | bool MopStI = HII->isMemOp(MI: I) || StoreI; |
| 1339 | bool MopStJ = HII->isMemOp(MI: J) || StoreJ; |
| 1340 | if (MopStI && MopStJ) |
| 1341 | return true; |
| 1342 | } |
| 1343 | |
| 1344 | return (StoreJ && HII->isDeallocRet(MI: I)) || (StoreI && HII->isDeallocRet(MI: J)); |
| 1345 | } |
| 1346 | |
| 1347 | // SUI is the current instruction that is outside of the current packet. |
| 1348 | // SUJ is the current instruction inside the current packet against which that |
| 1349 | // SUI will be packetized. |
| 1350 | bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) { |
| 1351 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 1352 | MachineInstr &I = *SUI->getInstr(); |
| 1353 | MachineInstr &J = *SUJ->getInstr(); |
| 1354 | |
| 1355 | // Clear IgnoreDepMIs when Packet starts. |
| 1356 | if (CurrentPacketMIs.size() == 1) |
| 1357 | IgnoreDepMIs.clear(); |
| 1358 | |
| 1359 | MachineBasicBlock::iterator II = I.getIterator(); |
| 1360 | |
| 1361 | // Solo instructions cannot go in the packet. |
| 1362 | assert(!isSoloInstruction(I) && "Unexpected solo instr!" ); |
| 1363 | |
| 1364 | if (cannotCoexist(MI: I, MJ: J)) |
| 1365 | return false; |
| 1366 | |
| 1367 | Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J); |
| 1368 | if (Dependence) |
| 1369 | return false; |
| 1370 | |
| 1371 | // Regmasks are not accounted for in the scheduling graph, so we need |
| 1372 | // to explicitly check for dependencies caused by them. They should only |
| 1373 | // appear on calls, so it's not too pessimistic to reject all regmask |
| 1374 | // dependencies. |
| 1375 | Dependence = hasRegMaskDependence(I, J); |
| 1376 | if (Dependence) |
| 1377 | return false; |
| 1378 | |
| 1379 | // Dual-store does not allow second store, if the first store is not |
| 1380 | // in SLOT0. New value store, new value jump, dealloc_return and memop |
| 1381 | // always take SLOT0. Arch spec 3.4.4.2. |
| 1382 | Dependence = hasDualStoreDependence(I, J); |
| 1383 | if (Dependence) |
| 1384 | return false; |
| 1385 | |
| 1386 | // If an instruction feeds new value jump, glue it. |
| 1387 | MachineBasicBlock::iterator NextMII = I.getIterator(); |
| 1388 | ++NextMII; |
| 1389 | if (NextMII != I.getParent()->end() && HII->isNewValueJump(MI: *NextMII)) { |
| 1390 | MachineInstr &NextMI = *NextMII; |
| 1391 | |
| 1392 | bool secondRegMatch = false; |
| 1393 | const MachineOperand &NOp0 = NextMI.getOperand(i: 0); |
| 1394 | const MachineOperand &NOp1 = NextMI.getOperand(i: 1); |
| 1395 | |
| 1396 | if (NOp1.isReg() && I.getOperand(i: 0).getReg() == NOp1.getReg()) |
| 1397 | secondRegMatch = true; |
| 1398 | |
| 1399 | for (MachineInstr *PI : CurrentPacketMIs) { |
| 1400 | // NVJ can not be part of the dual jump - Arch Spec: section 7.8. |
| 1401 | if (PI->isCall()) { |
| 1402 | Dependence = true; |
| 1403 | break; |
| 1404 | } |
| 1405 | // Validate: |
| 1406 | // 1. Packet does not have a store in it. |
| 1407 | // 2. If the first operand of the nvj is newified, and the second |
| 1408 | // operand is also a reg, it (second reg) is not defined in |
| 1409 | // the same packet. |
| 1410 | // 3. If the second operand of the nvj is newified, (which means |
| 1411 | // first operand is also a reg), first reg is not defined in |
| 1412 | // the same packet. |
| 1413 | if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() || |
| 1414 | HII->isLoopN(MI: *PI)) { |
| 1415 | Dependence = true; |
| 1416 | break; |
| 1417 | } |
| 1418 | // Check #2/#3. |
| 1419 | const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1; |
| 1420 | if (OpR.isReg() && PI->modifiesRegister(Reg: OpR.getReg(), TRI: HRI)) { |
| 1421 | Dependence = true; |
| 1422 | break; |
| 1423 | } |
| 1424 | } |
| 1425 | |
| 1426 | GlueToNewValueJump = true; |
| 1427 | if (Dependence) |
| 1428 | return false; |
| 1429 | } |
| 1430 | |
| 1431 | // There no dependency between a prolog instruction and its successor. |
| 1432 | if (!SUJ->isSucc(N: SUI)) |
| 1433 | return true; |
| 1434 | |
| 1435 | for (unsigned i = 0; i < SUJ->Succs.size(); ++i) { |
| 1436 | if (FoundSequentialDependence) |
| 1437 | break; |
| 1438 | |
| 1439 | if (SUJ->Succs[i].getSUnit() != SUI) |
| 1440 | continue; |
| 1441 | |
| 1442 | SDep::Kind DepType = SUJ->Succs[i].getKind(); |
| 1443 | // For direct calls: |
| 1444 | // Ignore register dependences for call instructions for packetization |
| 1445 | // purposes except for those due to r31 and predicate registers. |
| 1446 | // |
| 1447 | // For indirect calls: |
| 1448 | // Same as direct calls + check for true dependences to the register |
| 1449 | // used in the indirect call. |
| 1450 | // |
| 1451 | // We completely ignore Order dependences for call instructions. |
| 1452 | // |
| 1453 | // For returns: |
| 1454 | // Ignore register dependences for return instructions like jumpr, |
| 1455 | // dealloc return unless we have dependencies on the explicit uses |
| 1456 | // of the registers used by jumpr (like r31) or dealloc return |
| 1457 | // (like r29 or r30). |
| 1458 | unsigned DepReg = 0; |
| 1459 | const TargetRegisterClass *RC = nullptr; |
| 1460 | if (DepType == SDep::Data) { |
| 1461 | DepReg = SUJ->Succs[i].getReg(); |
| 1462 | RC = HRI->getMinimalPhysRegClass(Reg: DepReg); |
| 1463 | } |
| 1464 | |
| 1465 | if (I.isCall() || HII->isJumpR(MI: I) || I.isReturn() || HII->isTailCall(MI: I)) { |
| 1466 | if (!isRegDependence(DepType)) |
| 1467 | continue; |
| 1468 | if (!isCallDependent(MI: I, DepType, DepReg: SUJ->Succs[i].getReg())) |
| 1469 | continue; |
| 1470 | } |
| 1471 | |
| 1472 | if (DepType == SDep::Data) { |
| 1473 | if (canPromoteToDotCur(MI: J, PacketSU: SUJ, DepReg, MII&: II, RC)) |
| 1474 | if (promoteToDotCur(MI&: J, DepType, MII&: II, RC)) |
| 1475 | continue; |
| 1476 | } |
| 1477 | |
| 1478 | // Data dependence ok if we have load.cur. |
| 1479 | if (DepType == SDep::Data && HII->isDotCurInst(MI: J)) { |
| 1480 | if (HII->isHVXVec(MI: I)) |
| 1481 | continue; |
| 1482 | } |
| 1483 | |
| 1484 | // For instructions that can be promoted to dot-new, try to promote. |
| 1485 | if (DepType == SDep::Data) { |
| 1486 | if (canPromoteToDotNew(MI: I, PacketSU: SUJ, DepReg, MII&: II, RC)) { |
| 1487 | if (promoteToDotNew(MI&: I, DepType, MII&: II, RC)) { |
| 1488 | PromotedToDotNew = true; |
| 1489 | if (cannotCoexist(MI: I, MJ: J)) |
| 1490 | FoundSequentialDependence = true; |
| 1491 | continue; |
| 1492 | } |
| 1493 | } |
| 1494 | if (HII->isNewValueJump(MI: I)) |
| 1495 | continue; |
| 1496 | } |
| 1497 | |
| 1498 | // For predicated instructions, if the predicates are complements then |
| 1499 | // there can be no dependence. |
| 1500 | if (HII->isPredicated(MI: I) && HII->isPredicated(MI: J) && |
| 1501 | arePredicatesComplements(MI1&: I, MI2&: J)) { |
| 1502 | // Not always safe to do this translation. |
| 1503 | // DAG Builder attempts to reduce dependence edges using transitive |
| 1504 | // nature of dependencies. Here is an example: |
| 1505 | // |
| 1506 | // r0 = tfr_pt ... (1) |
| 1507 | // r0 = tfr_pf ... (2) |
| 1508 | // r0 = tfr_pt ... (3) |
| 1509 | // |
| 1510 | // There will be an output dependence between (1)->(2) and (2)->(3). |
| 1511 | // However, there is no dependence edge between (1)->(3). This results |
| 1512 | // in all 3 instructions going in the same packet. We ignore dependce |
| 1513 | // only once to avoid this situation. |
| 1514 | auto Itr = find(Range&: IgnoreDepMIs, Val: &J); |
| 1515 | if (Itr != IgnoreDepMIs.end()) { |
| 1516 | Dependence = true; |
| 1517 | return false; |
| 1518 | } |
| 1519 | IgnoreDepMIs.push_back(x: &I); |
| 1520 | continue; |
| 1521 | } |
| 1522 | |
| 1523 | // Ignore Order dependences between unconditional direct branches |
| 1524 | // and non-control-flow instructions. |
| 1525 | if (isDirectJump(MI: I) && !J.isBranch() && !J.isCall() && |
| 1526 | DepType == SDep::Order) |
| 1527 | continue; |
| 1528 | |
| 1529 | // Ignore all dependences for jumps except for true and output |
| 1530 | // dependences. |
| 1531 | if (I.isConditionalBranch() && DepType != SDep::Data && |
| 1532 | DepType != SDep::Output) |
| 1533 | continue; |
| 1534 | |
| 1535 | if (DepType == SDep::Output) { |
| 1536 | FoundSequentialDependence = true; |
| 1537 | break; |
| 1538 | } |
| 1539 | |
| 1540 | // For Order dependences: |
| 1541 | // 1. Volatile loads/stores can be packetized together, unless other |
| 1542 | // rules prevent is. |
| 1543 | // 2. Store followed by a load is not allowed. |
| 1544 | // 3. Store followed by a store is valid. |
| 1545 | // 4. Load followed by any memory operation is allowed. |
| 1546 | if (DepType == SDep::Order) { |
| 1547 | if (!PacketizeVolatiles) { |
| 1548 | bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef(); |
| 1549 | if (OrdRefs) { |
| 1550 | FoundSequentialDependence = true; |
| 1551 | break; |
| 1552 | } |
| 1553 | } |
| 1554 | // J is first, I is second. |
| 1555 | bool LoadJ = J.mayLoad(), StoreJ = J.mayStore(); |
| 1556 | bool LoadI = I.mayLoad(), StoreI = I.mayStore(); |
| 1557 | bool NVStoreJ = HII->isNewValueStore(MI: J); |
| 1558 | bool NVStoreI = HII->isNewValueStore(MI: I); |
| 1559 | bool IsVecJ = HII->isHVXVec(MI: J) || HII->isHLXVec(MI: J); |
| 1560 | bool IsVecI = HII->isHVXVec(MI: I) || HII->isHLXVec(MI: I); |
| 1561 | |
| 1562 | // Don't reorder the loads if there is an order dependence. This would |
| 1563 | // occur if the first instruction must go in slot0. |
| 1564 | if (LoadJ && LoadI && HII->isPureSlot0(MI: J)) { |
| 1565 | FoundSequentialDependence = true; |
| 1566 | break; |
| 1567 | } |
| 1568 | |
| 1569 | if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() && |
| 1570 | ((LoadJ && StoreI && !NVStoreI) || |
| 1571 | (StoreJ && LoadI && !NVStoreJ)) && |
| 1572 | (J.getOpcode() != Hexagon::S2_allocframe && |
| 1573 | I.getOpcode() != Hexagon::S2_allocframe) && |
| 1574 | (J.getOpcode() != Hexagon::L2_deallocframe && |
| 1575 | I.getOpcode() != Hexagon::L2_deallocframe) && |
| 1576 | (!HII->isMemOp(MI: J) && !HII->isMemOp(MI: I)) && (!IsVecJ && !IsVecI)) |
| 1577 | setmemShufDisabled(true); |
| 1578 | else |
| 1579 | if (StoreJ && LoadI && alias(MI1: J, MI2: I)) { |
| 1580 | FoundSequentialDependence = true; |
| 1581 | break; |
| 1582 | } |
| 1583 | |
| 1584 | if (!StoreJ) |
| 1585 | if (!LoadJ || (!LoadI && !StoreI)) { |
| 1586 | // If J is neither load nor store, assume a dependency. |
| 1587 | // If J is a load, but I is neither, also assume a dependency. |
| 1588 | FoundSequentialDependence = true; |
| 1589 | break; |
| 1590 | } |
| 1591 | // Store followed by store: not OK on V2. |
| 1592 | // Store followed by load: not OK on all. |
| 1593 | // Load followed by store: OK on all. |
| 1594 | // Load followed by load: OK on all. |
| 1595 | continue; |
| 1596 | } |
| 1597 | |
| 1598 | // Special case for ALLOCFRAME: even though there is dependency |
| 1599 | // between ALLOCFRAME and subsequent store, allow it to be packetized |
| 1600 | // in a same packet. This implies that the store is using the caller's |
| 1601 | // SP. Hence, offset needs to be updated accordingly. |
| 1602 | if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) { |
| 1603 | unsigned Opc = I.getOpcode(); |
| 1604 | switch (Opc) { |
| 1605 | case Hexagon::S2_storerd_io: |
| 1606 | case Hexagon::S2_storeri_io: |
| 1607 | case Hexagon::S2_storerh_io: |
| 1608 | case Hexagon::S2_storerb_io: |
| 1609 | if (I.getOperand(i: 0).getReg() == HRI->getStackRegister()) { |
| 1610 | // Since this store is to be glued with allocframe in the same |
| 1611 | // packet, it will use SP of the previous stack frame, i.e. |
| 1612 | // caller's SP. Therefore, we need to recalculate offset |
| 1613 | // according to this change. |
| 1614 | GlueAllocframeStore = useCallersSP(MI&: I); |
| 1615 | if (GlueAllocframeStore) |
| 1616 | continue; |
| 1617 | } |
| 1618 | break; |
| 1619 | default: |
| 1620 | break; |
| 1621 | } |
| 1622 | } |
| 1623 | |
| 1624 | // There are certain anti-dependencies that cannot be ignored. |
| 1625 | // Specifically: |
| 1626 | // J2_call ... implicit-def %r0 ; SUJ |
| 1627 | // R0 = ... ; SUI |
| 1628 | // Those cannot be packetized together, since the call will observe |
| 1629 | // the effect of the assignment to R0. |
| 1630 | if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) { |
| 1631 | // Check if I defines any volatile register. We should also check |
| 1632 | // registers that the call may read, but these happen to be a |
| 1633 | // subset of the volatile register set. |
| 1634 | for (const MachineOperand &Op : I.operands()) { |
| 1635 | if (Op.isReg() && Op.isDef()) { |
| 1636 | Register R = Op.getReg(); |
| 1637 | if (!J.readsRegister(Reg: R, TRI: HRI) && !J.modifiesRegister(Reg: R, TRI: HRI)) |
| 1638 | continue; |
| 1639 | } else if (!Op.isRegMask()) { |
| 1640 | // If I has a regmask assume dependency. |
| 1641 | continue; |
| 1642 | } |
| 1643 | FoundSequentialDependence = true; |
| 1644 | break; |
| 1645 | } |
| 1646 | } |
| 1647 | |
| 1648 | // Skip over remaining anti-dependences. Two instructions that are |
| 1649 | // anti-dependent can share a packet, since in most such cases all |
| 1650 | // operands are read before any modifications take place. |
| 1651 | // The exceptions are branch and call instructions, since they are |
| 1652 | // executed after all other instructions have completed (at least |
| 1653 | // conceptually). |
| 1654 | // Another exception is the HLX load pseudo, which expands to a temporary |
| 1655 | // load and a copy of the loaded value. An instruction it is anti-dependent |
| 1656 | // on would read the new value if both were in the same packet. |
| 1657 | if (DepType != SDep::Anti || I.getOpcode() == Hexagon::LX_L16b_pseudo) { |
| 1658 | FoundSequentialDependence = true; |
| 1659 | break; |
| 1660 | } |
| 1661 | } |
| 1662 | |
| 1663 | if (FoundSequentialDependence) { |
| 1664 | Dependence = true; |
| 1665 | return false; |
| 1666 | } |
| 1667 | |
| 1668 | return true; |
| 1669 | } |
| 1670 | |
| 1671 | bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) { |
| 1672 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 1673 | MachineInstr &I = *SUI->getInstr(); |
| 1674 | MachineInstr &J = *SUJ->getInstr(); |
| 1675 | |
| 1676 | bool Coexist = !cannotCoexist(MI: I, MJ: J); |
| 1677 | |
| 1678 | if (Coexist && !Dependence) |
| 1679 | return true; |
| 1680 | |
| 1681 | // Check if the instruction was promoted to a dot-new. If so, demote it |
| 1682 | // back into a dot-old. |
| 1683 | if (PromotedToDotNew) |
| 1684 | demoteToDotOld(MI&: I); |
| 1685 | |
| 1686 | cleanUpDotCur(); |
| 1687 | // Check if the instruction (must be a store) was glued with an allocframe |
| 1688 | // instruction. If so, restore its offset to its original value, i.e. use |
| 1689 | // current SP instead of caller's SP. |
| 1690 | if (GlueAllocframeStore) { |
| 1691 | useCalleesSP(MI&: I); |
| 1692 | GlueAllocframeStore = false; |
| 1693 | } |
| 1694 | |
| 1695 | if (ChangedOffset != INT64_MAX) |
| 1696 | undoChangedOffset(MI&: I); |
| 1697 | |
| 1698 | if (GlueToNewValueJump) { |
| 1699 | // Putting I and J together would prevent the new-value jump from being |
| 1700 | // packetized with the producer. In that case I and J must be separated. |
| 1701 | GlueToNewValueJump = false; |
| 1702 | return false; |
| 1703 | } |
| 1704 | |
| 1705 | if (!Coexist) |
| 1706 | return false; |
| 1707 | |
| 1708 | if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) { |
| 1709 | FoundSequentialDependence = false; |
| 1710 | Dependence = false; |
| 1711 | return true; |
| 1712 | } |
| 1713 | |
| 1714 | return false; |
| 1715 | } |
| 1716 | |
| 1717 | |
| 1718 | bool HexagonPacketizerList::foundLSInPacket() { |
| 1719 | bool FoundLoad = false; |
| 1720 | bool FoundStore = false; |
| 1721 | |
| 1722 | for (auto *MJ : CurrentPacketMIs) { |
| 1723 | unsigned Opc = MJ->getOpcode(); |
| 1724 | if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe) |
| 1725 | continue; |
| 1726 | if (HII->isMemOp(MI: *MJ)) |
| 1727 | continue; |
| 1728 | if (MJ->mayLoad()) |
| 1729 | FoundLoad = true; |
| 1730 | if (MJ->mayStore() && !HII->isNewValueStore(MI: *MJ)) |
| 1731 | FoundStore = true; |
| 1732 | } |
| 1733 | return FoundLoad && FoundStore; |
| 1734 | } |
| 1735 | |
| 1736 | |
| 1737 | MachineBasicBlock::iterator |
| 1738 | HexagonPacketizerList::addToPacket(MachineInstr &MI) { |
| 1739 | MachineBasicBlock::iterator MII = MI.getIterator(); |
| 1740 | MachineBasicBlock *MBB = MI.getParent(); |
| 1741 | |
| 1742 | if (CurrentPacketMIs.empty()) { |
| 1743 | PacketStalls = false; |
| 1744 | PacketStallCycles = 0; |
| 1745 | } |
| 1746 | PacketStalls |= producesStall(MI); |
| 1747 | PacketStallCycles = std::max(a: PacketStallCycles, b: calcStall(MI)); |
| 1748 | |
| 1749 | if (MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::FREEZE) { |
| 1750 | // Add to the packet to allow subsequent instructions to be checked |
| 1751 | // properly. |
| 1752 | CurrentPacketMIs.push_back(x: &MI); |
| 1753 | return MII; |
| 1754 | } |
| 1755 | assert(ResourceTracker->canReserveResources(MI)); |
| 1756 | |
| 1757 | MachineBasicBlock::iterator nextMII = MII; |
| 1758 | if (std::next(x: MII) != MI.getParent()->end()) { |
| 1759 | MachineInstr &nextMI = *++nextMII; |
| 1760 | if (HII->isActivateWeightPair(MIa: MI, MIb: nextMI)) { |
| 1761 | for (auto J : CurrentPacketMIs) { |
| 1762 | MachineInstr &JMI = *J; |
| 1763 | if (HII->isDependent(ProdMI: JMI, ConsMI: MI) || HII->isDependent(ProdMI: JMI, ConsMI: nextMI)) { |
| 1764 | endPacket(MBB, MI); |
| 1765 | break; |
| 1766 | } |
| 1767 | } |
| 1768 | ResourceTracker->reserveResources(MI); |
| 1769 | if (ResourceTracker->canReserveResources(MI&: nextMI)) |
| 1770 | ResourceTracker->reserveResources(MI&: nextMI); |
| 1771 | else { |
| 1772 | endPacket(MBB, MI); |
| 1773 | assert(ResourceTracker->canReserveResources(MI)); |
| 1774 | ResourceTracker->reserveResources(MI); |
| 1775 | assert(ResourceTracker->canReserveResources(nextMI)); |
| 1776 | ResourceTracker->reserveResources(MI&: nextMI); |
| 1777 | } |
| 1778 | CurrentPacketMIs.push_back(x: &MI); |
| 1779 | CurrentPacketMIs.push_back(x: &nextMI); |
| 1780 | return nextMII; |
| 1781 | } |
| 1782 | } |
| 1783 | |
| 1784 | bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI); |
| 1785 | bool Good = true; |
| 1786 | |
| 1787 | if (GlueToNewValueJump) { |
| 1788 | MachineInstr &NvjMI = *++MII; |
| 1789 | // We need to put both instructions in the same packet: MI and NvjMI. |
| 1790 | // Either of them can require a constant extender. Try to add both to |
| 1791 | // the current packet, and if that fails, end the packet and start a |
| 1792 | // new one. |
| 1793 | ResourceTracker->reserveResources(MI); |
| 1794 | if (ExtMI) |
| 1795 | Good = tryAllocateResourcesForConstExt(Reserve: true); |
| 1796 | |
| 1797 | bool ExtNvjMI = HII->isExtended(MI: NvjMI) || HII->isConstExtended(MI: NvjMI); |
| 1798 | if (Good) { |
| 1799 | if (ResourceTracker->canReserveResources(MI&: NvjMI)) |
| 1800 | ResourceTracker->reserveResources(MI&: NvjMI); |
| 1801 | else |
| 1802 | Good = false; |
| 1803 | } |
| 1804 | if (Good && ExtNvjMI) |
| 1805 | Good = tryAllocateResourcesForConstExt(Reserve: true); |
| 1806 | |
| 1807 | if (!Good) { |
| 1808 | endPacket(MBB, MI); |
| 1809 | assert(ResourceTracker->canReserveResources(MI)); |
| 1810 | ResourceTracker->reserveResources(MI); |
| 1811 | if (ExtMI) { |
| 1812 | assert(canReserveResourcesForConstExt()); |
| 1813 | tryAllocateResourcesForConstExt(Reserve: true); |
| 1814 | } |
| 1815 | assert(ResourceTracker->canReserveResources(NvjMI)); |
| 1816 | ResourceTracker->reserveResources(MI&: NvjMI); |
| 1817 | if (ExtNvjMI) { |
| 1818 | assert(canReserveResourcesForConstExt()); |
| 1819 | reserveResourcesForConstExt(); |
| 1820 | } |
| 1821 | } |
| 1822 | CurrentPacketMIs.push_back(x: &MI); |
| 1823 | CurrentPacketMIs.push_back(x: &NvjMI); |
| 1824 | return MII; |
| 1825 | } |
| 1826 | |
| 1827 | ResourceTracker->reserveResources(MI); |
| 1828 | if (ExtMI && !tryAllocateResourcesForConstExt(Reserve: true)) { |
| 1829 | endPacket(MBB, MI); |
| 1830 | if (PromotedToDotNew) |
| 1831 | demoteToDotOld(MI); |
| 1832 | if (GlueAllocframeStore) { |
| 1833 | useCalleesSP(MI); |
| 1834 | GlueAllocframeStore = false; |
| 1835 | } |
| 1836 | ResourceTracker->reserveResources(MI); |
| 1837 | reserveResourcesForConstExt(); |
| 1838 | } |
| 1839 | |
| 1840 | CurrentPacketMIs.push_back(x: &MI); |
| 1841 | return MII; |
| 1842 | } |
| 1843 | |
| 1844 | void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB, |
| 1845 | MachineBasicBlock::iterator EndMI) { |
| 1846 | // Replace VLIWPacketizerList::endPacket(MBB, EndMI). |
| 1847 | LLVM_DEBUG({ |
| 1848 | if (!CurrentPacketMIs.empty()) { |
| 1849 | dbgs() << "Finalizing packet:\n" ; |
| 1850 | unsigned Idx = 0; |
| 1851 | for (MachineInstr *MI : CurrentPacketMIs) { |
| 1852 | unsigned R = ResourceTracker->getUsedResources(Idx++); |
| 1853 | dbgs() << " * [res:0x" << utohexstr(R) << "] " << *MI; |
| 1854 | } |
| 1855 | } |
| 1856 | }); |
| 1857 | |
| 1858 | bool memShufDisabled = getmemShufDisabled(); |
| 1859 | if (memShufDisabled && !foundLSInPacket()) { |
| 1860 | setmemShufDisabled(false); |
| 1861 | LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n" ); |
| 1862 | } |
| 1863 | memShufDisabled = getmemShufDisabled(); |
| 1864 | |
| 1865 | OldPacketMIs.clear(); |
| 1866 | for (MachineInstr *MI : CurrentPacketMIs) { |
| 1867 | MachineBasicBlock::instr_iterator NextMI = std::next(x: MI->getIterator()); |
| 1868 | for (auto &I : make_range(x: HII->expandVGatherPseudo(MI&: *MI), y: NextMI)) |
| 1869 | OldPacketMIs.push_back(x: &I); |
| 1870 | } |
| 1871 | CurrentPacketMIs.clear(); |
| 1872 | |
| 1873 | if (OldPacketMIs.size() > 1) { |
| 1874 | MachineBasicBlock::instr_iterator FirstMI(OldPacketMIs.front()); |
| 1875 | MachineBasicBlock::instr_iterator LastMI(EndMI.getInstrIterator()); |
| 1876 | finalizeBundle(MBB&: *MBB, FirstMI, LastMI); |
| 1877 | auto BundleMII = std::prev(x: FirstMI); |
| 1878 | if (memShufDisabled) |
| 1879 | HII->setBundleNoShuf(BundleMII); |
| 1880 | |
| 1881 | setmemShufDisabled(false); |
| 1882 | } |
| 1883 | |
| 1884 | PacketHasDuplex = false; |
| 1885 | PacketHasSLOT0OnlyInsn = false; |
| 1886 | ResourceTracker->clearResources(); |
| 1887 | LLVM_DEBUG(dbgs() << "End packet\n" ); |
| 1888 | } |
| 1889 | |
| 1890 | bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) { |
| 1891 | if (Minimal) |
| 1892 | return false; |
| 1893 | |
| 1894 | if (producesStall(MI)) |
| 1895 | return false; |
| 1896 | |
| 1897 | // If TinyCore with Duplexes is enabled, check if this MI can form a Duplex |
| 1898 | // with any other instruction in the existing packet. |
| 1899 | auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>(); |
| 1900 | // Constraint 1: Only one duplex allowed per packet. |
| 1901 | // Constraint 2: Consider duplex checks only if there is at least one |
| 1902 | // instruction in a packet. |
| 1903 | // Constraint 3: If one of the existing instructions in the packet has a |
| 1904 | // SLOT0 only instruction that can not be duplexed, do not attempt to form |
| 1905 | // duplexes. (TODO: This will invalidate the L4_return* instructions to form a |
| 1906 | // duplex) |
| 1907 | if (HST.isTinyCoreWithDuplex() && CurrentPacketMIs.size() > 0 && |
| 1908 | !PacketHasDuplex) { |
| 1909 | // Check for SLOT0 only non-duplexable instruction in packet. |
| 1910 | for (auto &MJ : CurrentPacketMIs) |
| 1911 | PacketHasSLOT0OnlyInsn |= HII->isPureSlot0(MI: *MJ); |
| 1912 | // Get the Big Core Opcode (dup_*). |
| 1913 | int Opcode = HII->getDuplexOpcode(MI, ForBigCore: false); |
| 1914 | if (Opcode >= 0) { |
| 1915 | // We now have an instruction that can be duplexed. |
| 1916 | for (auto &MJ : CurrentPacketMIs) { |
| 1917 | if (HII->isDuplexPair(MIa: MI, MIb: *MJ) && !PacketHasSLOT0OnlyInsn) { |
| 1918 | PacketHasDuplex = true; |
| 1919 | return true; |
| 1920 | } |
| 1921 | } |
| 1922 | // If it can not be duplexed, check if there is a valid transition in DFA |
| 1923 | // with the original opcode. |
| 1924 | MachineInstr &MIRef = const_cast<MachineInstr &>(MI); |
| 1925 | MIRef.setDesc(HII->get(Opcode)); |
| 1926 | return ResourceTracker->canReserveResources(MI&: MIRef); |
| 1927 | } |
| 1928 | } |
| 1929 | |
| 1930 | // An HLX load/store pseudo expands to two instructions in its packet. With |
| 1931 | // one more instruction in the packet, there is no room left for a |
| 1932 | // constant-extended instruction and its extender within the four-word |
| 1933 | // packet limit. |
| 1934 | if (HST.useHLXOps() && CurrentPacketMIs.size() > 1) { |
| 1935 | auto IsHLXLdStPseudo = [this](const MachineInstr &I) { |
| 1936 | return HII->isTmpLoadPseudo(MI: I) || HII->isNewStorePseudo(MI: I); |
| 1937 | }; |
| 1938 | bool HasConstExt = HII->isConstExtended(MI); |
| 1939 | bool HasPseudo = IsHLXLdStPseudo(MI); |
| 1940 | for (const MachineInstr *MJ : CurrentPacketMIs) { |
| 1941 | HasConstExt |= HII->isConstExtended(MI: *MJ); |
| 1942 | HasPseudo |= IsHLXLdStPseudo(*MJ); |
| 1943 | } |
| 1944 | if (HasConstExt && HasPseudo) |
| 1945 | return false; |
| 1946 | } |
| 1947 | |
| 1948 | return true; |
| 1949 | } |
| 1950 | |
| 1951 | // V60 forward scheduling. |
| 1952 | unsigned int HexagonPacketizerList::calcStall(const MachineInstr &I) { |
| 1953 | // Check whether the previous packet is in a different loop. If this is the |
| 1954 | // case, there is little point in trying to avoid a stall because that would |
| 1955 | // favor the rare case (loop entry) over the common case (loop iteration). |
| 1956 | // |
| 1957 | // TODO: We should really be able to check all the incoming edges if this is |
| 1958 | // the first packet in a basic block, so we can avoid stalls from the loop |
| 1959 | // backedge. |
| 1960 | if (!OldPacketMIs.empty()) { |
| 1961 | auto *OldBB = OldPacketMIs.front()->getParent(); |
| 1962 | auto *ThisBB = I.getParent(); |
| 1963 | if (MLI->getLoopFor(BB: OldBB) != MLI->getLoopFor(BB: ThisBB)) |
| 1964 | return 0; |
| 1965 | } |
| 1966 | |
| 1967 | SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)]; |
| 1968 | if (!SUI) |
| 1969 | return 0; |
| 1970 | |
| 1971 | // If the latency is 0 and there is a data dependence between this |
| 1972 | // instruction and any instruction in the current packet, we disregard any |
| 1973 | // potential stalls due to the instructions in the previous packet. Most of |
| 1974 | // the instruction pairs that can go together in the same packet have 0 |
| 1975 | // latency between them. The exceptions are |
| 1976 | // 1. NewValueJumps as they're generated much later and the latencies can't |
| 1977 | // be changed at that point. |
| 1978 | // 2. .cur instructions, if its consumer has a 0 latency successor (such as |
| 1979 | // .new). In this case, the latency between .cur and the consumer stays |
| 1980 | // non-zero even though we can have both .cur and .new in the same packet. |
| 1981 | // Changing the latency to 0 is not an option as it causes software pipeliner |
| 1982 | // to not pipeline in some cases. |
| 1983 | |
| 1984 | // For Example: |
| 1985 | // { |
| 1986 | // I1: v6.cur = vmem(r0++#1) |
| 1987 | // I2: v7 = valign(v6,v4,r2) |
| 1988 | // I3: vmem(r5++#1) = v7.new |
| 1989 | // } |
| 1990 | // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2. |
| 1991 | |
| 1992 | for (auto *J : CurrentPacketMIs) { |
| 1993 | SUnit *SUJ = MIToSUnit[J]; |
| 1994 | for (auto &Pred : SUI->Preds) |
| 1995 | if (Pred.getSUnit() == SUJ) |
| 1996 | if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) || |
| 1997 | HII->isNewValueJump(MI: I) || HII->isToBeScheduledASAP(MI1: *J, MI2: I)) |
| 1998 | return 0; |
| 1999 | } |
| 2000 | |
| 2001 | // Check if the latency is greater than one between this instruction and any |
| 2002 | // instruction in the previous packet. |
| 2003 | for (auto *J : OldPacketMIs) { |
| 2004 | SUnit *SUJ = MIToSUnit[J]; |
| 2005 | for (auto &Pred : SUI->Preds) |
| 2006 | if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1) |
| 2007 | return Pred.getLatency(); |
| 2008 | } |
| 2009 | |
| 2010 | return 0; |
| 2011 | } |
| 2012 | |
| 2013 | bool HexagonPacketizerList::producesStall(const MachineInstr &I) { |
| 2014 | unsigned int Latency = calcStall(I); |
| 2015 | if (Latency == 0) |
| 2016 | return false; |
| 2017 | // Ignore stall unless it stalls more than previous instruction in packet |
| 2018 | if (PacketStalls) |
| 2019 | return Latency > PacketStallCycles; |
| 2020 | return true; |
| 2021 | } |
| 2022 | |
| 2023 | //===----------------------------------------------------------------------===// |
| 2024 | // Public Constructor Functions |
| 2025 | //===----------------------------------------------------------------------===// |
| 2026 | |
| 2027 | FunctionPass *llvm::createHexagonPacketizer(bool Minimal) { |
| 2028 | return new HexagonPacketizer(Minimal); |
| 2029 | } |
| 2030 | |