| 1 | //===- HexagonFrameLowering.cpp - Define frame lowering -------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | |
| 10 | #include "HexagonFrameLowering.h" |
| 11 | #include "HexagonBlockRanges.h" |
| 12 | #include "HexagonISelLowering.h" |
| 13 | #include "HexagonInstrInfo.h" |
| 14 | #include "HexagonMachineFunctionInfo.h" |
| 15 | #include "HexagonRegisterInfo.h" |
| 16 | #include "HexagonSubtarget.h" |
| 17 | #include "HexagonTargetMachine.h" |
| 18 | #include "MCTargetDesc/HexagonBaseInfo.h" |
| 19 | #include "llvm/ADT/BitVector.h" |
| 20 | #include "llvm/ADT/DenseMap.h" |
| 21 | #include "llvm/ADT/PostOrderIterator.h" |
| 22 | #include "llvm/ADT/SetVector.h" |
| 23 | #include "llvm/ADT/SmallSet.h" |
| 24 | #include "llvm/ADT/SmallVector.h" |
| 25 | #include "llvm/BinaryFormat/Dwarf.h" |
| 26 | #include "llvm/CodeGen/CFIInstBuilder.h" |
| 27 | #include "llvm/CodeGen/LivePhysRegs.h" |
| 28 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 29 | #include "llvm/CodeGen/MachineDominators.h" |
| 30 | #include "llvm/CodeGen/MachineFrameInfo.h" |
| 31 | #include "llvm/CodeGen/MachineFunction.h" |
| 32 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 33 | #include "llvm/CodeGen/MachineInstr.h" |
| 34 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 35 | #include "llvm/CodeGen/MachineMemOperand.h" |
| 36 | #include "llvm/CodeGen/MachineModuleInfo.h" |
| 37 | #include "llvm/CodeGen/MachineOperand.h" |
| 38 | #include "llvm/CodeGen/MachinePostDominators.h" |
| 39 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 40 | #include "llvm/CodeGen/PseudoSourceValue.h" |
| 41 | #include "llvm/CodeGen/RegisterScavenging.h" |
| 42 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
| 43 | #include "llvm/IR/Attributes.h" |
| 44 | #include "llvm/IR/DebugLoc.h" |
| 45 | #include "llvm/IR/DerivedTypes.h" |
| 46 | #include "llvm/IR/DiagnosticInfo.h" |
| 47 | #include "llvm/IR/Function.h" |
| 48 | #include "llvm/MC/MCDwarf.h" |
| 49 | #include "llvm/MC/MCRegisterInfo.h" |
| 50 | #include "llvm/Pass.h" |
| 51 | #include "llvm/Support/CodeGen.h" |
| 52 | #include "llvm/Support/CommandLine.h" |
| 53 | #include "llvm/Support/Compiler.h" |
| 54 | #include "llvm/Support/Debug.h" |
| 55 | #include "llvm/Support/ErrorHandling.h" |
| 56 | #include "llvm/Support/LEB128.h" |
| 57 | #include "llvm/Support/MathExtras.h" |
| 58 | #include "llvm/Support/raw_ostream.h" |
| 59 | #include "llvm/Target/TargetMachine.h" |
| 60 | #include <algorithm> |
| 61 | #include <cassert> |
| 62 | #include <cstdint> |
| 63 | #include <iterator> |
| 64 | #include <limits> |
| 65 | #include <map> |
| 66 | #include <optional> |
| 67 | #include <utility> |
| 68 | #include <vector> |
| 69 | |
| 70 | #define DEBUG_TYPE "hexagon-pei" |
| 71 | |
| 72 | static bool containsHLXObject(const llvm::HexagonSubtarget &HST, |
| 73 | llvm::Type *Ty) { |
| 74 | if (Ty->isVectorTy()) |
| 75 | return HST.isTypeForHLX(VecTy: Ty); |
| 76 | if (auto *ArrayTy = llvm::dyn_cast<llvm::ArrayType>(Val: Ty)) |
| 77 | return containsHLXObject(HST, Ty: ArrayTy->getElementType()); |
| 78 | if (auto *StructTy = llvm::dyn_cast<llvm::StructType>(Val: Ty)) |
| 79 | return llvm::any_of(Range: StructTy->elements(), P: [&](llvm::Type *EltTy) { |
| 80 | return containsHLXObject(HST, Ty: EltTy); |
| 81 | }); |
| 82 | return false; |
| 83 | } |
| 84 | |
| 85 | // Hexagon stack frame layout as defined by the ABI: |
| 86 | // |
| 87 | // Incoming arguments |
| 88 | // passed via stack |
| 89 | // | |
| 90 | // | |
| 91 | // SP during function's FP during function's | |
| 92 | // +-- runtime (top of stack) runtime (bottom) --+ | |
| 93 | // | | | |
| 94 | // --++---------------------+------------------+-----------------++-+------- |
| 95 | // | parameter area for | variable-size | fixed-size |LR| arg |
| 96 | // | called functions | local objects | local objects |FP| |
| 97 | // --+----------------------+------------------+-----------------+--+------- |
| 98 | // <- size known -> <- size unknown -> <- size known -> |
| 99 | // |
| 100 | // Low address High address |
| 101 | // |
| 102 | // <--- stack growth |
| 103 | // |
| 104 | // |
| 105 | // - In any circumstances, the outgoing function arguments are always accessi- |
| 106 | // ble using the SP, and the incoming arguments are accessible using the FP. |
| 107 | // - If the local objects are not aligned, they can always be accessed using |
| 108 | // the FP. |
| 109 | // - If there are no variable-sized objects, the local objects can always be |
| 110 | // accessed using the SP, regardless whether they are aligned or not. (The |
| 111 | // alignment padding will be at the bottom of the stack (highest address), |
| 112 | // and so the offset with respect to the SP will be known at the compile- |
| 113 | // -time.) |
| 114 | // |
| 115 | // The only complication occurs if there are both, local aligned objects, and |
| 116 | // dynamically allocated (variable-sized) objects. The alignment pad will be |
| 117 | // placed between the FP and the local objects, thus preventing the use of the |
| 118 | // FP to access the local objects. At the same time, the variable-sized objects |
| 119 | // will be between the SP and the local objects, thus introducing an unknown |
| 120 | // distance from the SP to the locals. |
| 121 | // |
| 122 | // To avoid this problem, a new register is created that holds the aligned |
| 123 | // address of the bottom of the stack, referred in the sources as AP (aligned |
| 124 | // pointer). The AP will be equal to "FP-p", where "p" is the smallest pad |
| 125 | // that aligns AP to the required boundary (a maximum of the alignments of |
| 126 | // all stack objects, fixed- and variable-sized). All local objects[1] will |
| 127 | // then use AP as the base pointer. |
| 128 | // [1] The exception is with "fixed" stack objects. "Fixed" stack objects get |
| 129 | // their name from being allocated at fixed locations on the stack, relative |
| 130 | // to the FP. In the presence of dynamic allocation and local alignment, such |
| 131 | // objects can only be accessed through the FP. |
| 132 | // |
| 133 | // Illustration of the AP: |
| 134 | // FP --+ |
| 135 | // | |
| 136 | // ---------------+---------------------+-----+-----------------------++-+-- |
| 137 | // Rest of the | Local stack objects | Pad | Fixed stack objects |LR| |
| 138 | // stack frame | (aligned) | | (CSR, spills, etc.) |FP| |
| 139 | // ---------------+---------------------+-----+-----------------+-----+--+-- |
| 140 | // |<-- Multiple of the -->| |
| 141 | // stack alignment +-- AP |
| 142 | // |
| 143 | // The AP is set up at the beginning of the function. Since it is not a dedi- |
| 144 | // cated (reserved) register, it needs to be kept live throughout the function |
| 145 | // to be available as the base register for local object accesses. |
| 146 | // Normally, an address of a stack objects is obtained by a pseudo-instruction |
| 147 | // PS_fi. To access local objects with the AP register present, a different |
| 148 | // pseudo-instruction needs to be used: PS_fia. The PS_fia takes one extra |
| 149 | // argument compared to PS_fi: the first input register is the AP register. |
| 150 | // This keeps the register live between its definition and its uses. |
| 151 | |
| 152 | // The AP register is originally set up using pseudo-instruction PS_aligna: |
| 153 | // AP = PS_aligna A |
| 154 | // where |
| 155 | // A - required stack alignment |
| 156 | // The alignment value must be the maximum of all alignments required by |
| 157 | // any stack object. |
| 158 | |
| 159 | // The dynamic allocation uses a pseudo-instruction PS_alloca: |
| 160 | // Rd = PS_alloca Rs, A |
| 161 | // where |
| 162 | // Rd - address of the allocated space |
| 163 | // Rs - minimum size (the actual allocated can be larger to accommodate |
| 164 | // alignment) |
| 165 | // A - required alignment |
| 166 | |
| 167 | using namespace llvm; |
| 168 | |
| 169 | static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, |
| 170 | MachineBasicBlock::iterator MI, |
| 171 | const DebugLoc &DL) { |
| 172 | if (!MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) |
| 173 | return; |
| 174 | |
| 175 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 176 | // Hexagon saves LR (R31) via allocframe. If there is no frame, LR is |
| 177 | // not on the regular stack and does not need shadow-stack protection. |
| 178 | if (!HST.getFrameLowering()->hasFP(MF)) |
| 179 | return; |
| 180 | |
| 181 | // The shadow call stack pointer has to survive arbitrary calls, so it is |
| 182 | // always one of the callee-saved registers R16-R27 (Hexagon ABI, "Register |
| 183 | // usage across calls"). It must also be reserved: besides keeping the |
| 184 | // register allocator away from it, reserving it keeps it out of the |
| 185 | // callee-saved set, so it is never spilled and restored as an ordinary |
| 186 | // callee-saved register - which would leave the epilogue below reading the |
| 187 | // *caller's* shadow-stack slot. The spill stubs are handled separately in |
| 188 | // useSpillFunction()/useRestoreFunction(). |
| 189 | Register SCSPReg = HST.getSCSPReg(); |
| 190 | const auto &HRI = *HST.getRegisterInfo(); |
| 191 | if (!HST.isRegisterReservedByUser(i: SCSPReg)) |
| 192 | // Lower-cased to match the spelling of the -ffixed-<reg> flag the user |
| 193 | // needs to pass; TRI names the register "R18". |
| 194 | report_fatal_error(reason: Twine("Must reserve " ) + |
| 195 | StringRef(HRI.getName(RegNo: SCSPReg)).lower() + |
| 196 | " to use shadow call stack on Hexagon" ); |
| 197 | |
| 198 | const auto &HII = *HST.getInstrInfo(); |
| 199 | |
| 200 | // SCSPReg = add(SCSPReg, #4) |
| 201 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SCSPReg) |
| 202 | .addReg(RegNo: SCSPReg) |
| 203 | .addImm(Val: 4) |
| 204 | .setMIFlag(MachineInstr::FrameSetup); |
| 205 | // memw(SCSPReg + #-4) = r31 |
| 206 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::S2_storeri_io)) |
| 207 | .addReg(RegNo: SCSPReg) |
| 208 | .addImm(Val: -4) |
| 209 | .addReg(RegNo: Hexagon::R31) |
| 210 | .setMIFlag(MachineInstr::FrameSetup); |
| 211 | |
| 212 | MBB.addLiveIn(PhysReg: SCSPReg); |
| 213 | |
| 214 | if (!MF.needsFrameMoves()) |
| 215 | return; |
| 216 | |
| 217 | // CFI: DW_CFA_val_expression for the SCS register, DW_OP_bregN -4 |
| 218 | // Tells the unwinder that the SCS register at entry = current value - 4. |
| 219 | unsigned DwarfSCSReg = HRI.getDwarfRegNum(Reg: SCSPReg, /*IsEH=*/isEH: true); |
| 220 | // DW_OP_breg0..DW_OP_breg31 (0x70..0x8f) are 32 opcodes indexed by |
| 221 | // register number, so the register number must fit in [0, 31]. |
| 222 | assert(DwarfSCSReg < 32 && "SCS register should be < 32" ); |
| 223 | const char CFIInst[] = { |
| 224 | (char)dwarf::DW_CFA_val_expression, |
| 225 | (char)DwarfSCSReg, |
| 226 | 2, // expression length |
| 227 | (char)(unsigned)(dwarf::DW_OP_breg0 + DwarfSCSReg), |
| 228 | (char)(-4 & 0x7f), // SLEB128 -4 |
| 229 | }; |
| 230 | CFIInstBuilder(MBB, MI, MachineInstr::FrameSetup) |
| 231 | .buildEscape(Bytes: StringRef(CFIInst, sizeof(CFIInst))); |
| 232 | } |
| 233 | |
| 234 | static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, |
| 235 | MachineBasicBlock::iterator MI, |
| 236 | const DebugLoc &DL) { |
| 237 | if (!MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) |
| 238 | return; |
| 239 | |
| 240 | // hasFP() is true at both call sites: the non-vararg path in |
| 241 | // insertEpilogueInBlock returns early when !hasFP(), and the vararg+musl |
| 242 | // path is inside the hasFP() branch. Check defensively. |
| 243 | if (!MF.getSubtarget<HexagonSubtarget>().getFrameLowering()->hasFP(MF)) |
| 244 | report_fatal_error(reason: "SCS epilogue requires a frame" ); |
| 245 | |
| 246 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 247 | Register SCSPReg = HST.getSCSPReg(); |
| 248 | const auto &HII = *HST.getInstrInfo(); |
| 249 | |
| 250 | // r31 = memw(SCSPReg + #-4) |
| 251 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::L2_loadri_io), DestReg: Hexagon::R31) |
| 252 | .addReg(RegNo: SCSPReg) |
| 253 | .addImm(Val: -4) |
| 254 | .setMIFlag(MachineInstr::FrameDestroy); |
| 255 | // SCSPReg = add(SCSPReg, #-4) |
| 256 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SCSPReg) |
| 257 | .addReg(RegNo: SCSPReg) |
| 258 | .addImm(Val: -4) |
| 259 | .setMIFlag(MachineInstr::FrameDestroy); |
| 260 | |
| 261 | if (MF.needsFrameMoves()) |
| 262 | CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(Reg: SCSPReg); |
| 263 | } |
| 264 | |
| 265 | static cl::opt<bool> DisableDeallocRet("disable-hexagon-dealloc-ret" , |
| 266 | cl::Hidden, cl::desc("Disable Dealloc Return for Hexagon target" )); |
| 267 | |
| 268 | static cl::opt<unsigned> |
| 269 | NumberScavengerSlots("number-scavenger-slots" , cl::Hidden, |
| 270 | cl::desc("Set the number of scavenger slots" ), |
| 271 | cl::init(Val: 2)); |
| 272 | |
| 273 | static cl::opt<int> |
| 274 | SpillFuncThreshold("spill-func-threshold" , cl::Hidden, |
| 275 | cl::desc("Specify O2(not Os) spill func threshold" ), |
| 276 | cl::init(Val: 6)); |
| 277 | |
| 278 | static cl::opt<int> |
| 279 | SpillFuncThresholdOs("spill-func-threshold-Os" , cl::Hidden, |
| 280 | cl::desc("Specify Os spill func threshold" ), |
| 281 | cl::init(Val: 1)); |
| 282 | |
| 283 | static cl::opt<bool> EnableStackOVFSanitizer( |
| 284 | "enable-stackovf-sanitizer" , cl::Hidden, |
| 285 | cl::desc("Enable runtime checks for stack overflow." ), cl::init(Val: false)); |
| 286 | |
| 287 | static cl::opt<bool> |
| 288 | EnableShrinkWrapping("hexagon-shrink-frame" , cl::init(Val: true), cl::Hidden, |
| 289 | cl::desc("Enable stack frame shrink wrapping" )); |
| 290 | |
| 291 | static cl::opt<unsigned> |
| 292 | ShrinkLimit("shrink-frame-limit" , |
| 293 | cl::init(Val: std::numeric_limits<unsigned>::max()), cl::Hidden, |
| 294 | cl::desc("Max count of stack frame shrink-wraps" )); |
| 295 | |
| 296 | static cl::opt<bool> |
| 297 | EnableSaveRestoreLong("enable-save-restore-long" , cl::Hidden, |
| 298 | cl::desc("Enable long calls for save-restore stubs." ), |
| 299 | cl::init(Val: false)); |
| 300 | |
| 301 | static cl::opt<unsigned> HLXVectorStackLimit( |
| 302 | "hlx-vector-stack-limit" , cl::Hidden, |
| 303 | cl::init(Val: std::numeric_limits<unsigned>::max()), |
| 304 | cl::desc("Maximum HLX vector (VTCM) stack frame size in bytes" )); |
| 305 | |
| 306 | static cl::opt<bool> EliminateFramePointer("hexagon-fp-elim" , cl::init(Val: true), |
| 307 | cl::Hidden, cl::desc("Refrain from using FP whenever possible" )); |
| 308 | |
| 309 | static cl::opt<bool> OptimizeSpillSlots("hexagon-opt-spill" , cl::Hidden, |
| 310 | cl::init(Val: true), cl::desc("Optimize spill slots" )); |
| 311 | |
| 312 | static cl::opt<bool> SaveParentVSP( |
| 313 | "hexagon-save-parent-vsp" , cl::Hidden, cl::init(Val: false), |
| 314 | cl::desc("Save the caller's VSP on the stack in functions with an HLX " |
| 315 | "vector stack frame, for debuggers" )); |
| 316 | |
| 317 | #ifndef NDEBUG |
| 318 | static cl::opt<unsigned> SpillOptMax("spill-opt-max" , cl::Hidden, |
| 319 | cl::init(std::numeric_limits<unsigned>::max())); |
| 320 | static unsigned SpillOptCount = 0; |
| 321 | #endif |
| 322 | |
| 323 | namespace { |
| 324 | |
| 325 | class HexagonCallFrameInformation : public MachineFunctionPass { |
| 326 | public: |
| 327 | static char ID; |
| 328 | |
| 329 | HexagonCallFrameInformation() : MachineFunctionPass(ID) {} |
| 330 | |
| 331 | bool runOnMachineFunction(MachineFunction &MF) override; |
| 332 | |
| 333 | MachineFunctionProperties getRequiredProperties() const override { |
| 334 | return MachineFunctionProperties().setNoVRegs(); |
| 335 | } |
| 336 | }; |
| 337 | |
| 338 | char HexagonCallFrameInformation::ID = 0; |
| 339 | |
| 340 | } // end anonymous namespace |
| 341 | |
| 342 | bool HexagonCallFrameInformation::runOnMachineFunction(MachineFunction &MF) { |
| 343 | auto &HFI = *MF.getSubtarget<HexagonSubtarget>().getFrameLowering(); |
| 344 | bool NeedCFI = MF.needsFrameMoves(); |
| 345 | |
| 346 | if (!NeedCFI) |
| 347 | return false; |
| 348 | HFI.insertCFIInstructions(MF); |
| 349 | return true; |
| 350 | } |
| 351 | |
| 352 | INITIALIZE_PASS(HexagonCallFrameInformation, "hexagon-cfi" , |
| 353 | "Hexagon call frame information" , false, false) |
| 354 | |
| 355 | FunctionPass *llvm::createHexagonCallFrameInformation() { |
| 356 | return new HexagonCallFrameInformation(); |
| 357 | } |
| 358 | |
| 359 | /// Map a register pair Reg to the subregister that has the greater "number", |
| 360 | /// i.e. D3 (aka R7:6) will be mapped to R7, etc. |
| 361 | static Register getMax32BitSubRegister(Register Reg, |
| 362 | const TargetRegisterInfo &TRI, |
| 363 | bool hireg = true) { |
| 364 | if (Reg < Hexagon::D0 || Reg > Hexagon::D15) |
| 365 | return Reg; |
| 366 | |
| 367 | Register RegNo = 0; |
| 368 | for (MCPhysReg SubReg : TRI.subregs(Reg)) { |
| 369 | if (hireg) { |
| 370 | if (SubReg > RegNo) |
| 371 | RegNo = SubReg; |
| 372 | } else { |
| 373 | if (!RegNo || SubReg < RegNo) |
| 374 | RegNo = SubReg; |
| 375 | } |
| 376 | } |
| 377 | return RegNo; |
| 378 | } |
| 379 | |
| 380 | /// Returns the callee saved register with the largest id in the vector. |
| 381 | static Register getMaxCalleeSavedReg(ArrayRef<CalleeSavedInfo> CSI, |
| 382 | const TargetRegisterInfo &TRI) { |
| 383 | static_assert(Hexagon::R1 > 0, |
| 384 | "Assume physical registers are encoded as positive integers" ); |
| 385 | if (CSI.empty()) |
| 386 | return 0; |
| 387 | |
| 388 | Register Max = getMax32BitSubRegister(Reg: CSI[0].getReg(), TRI); |
| 389 | for (unsigned I = 1, E = CSI.size(); I < E; ++I) { |
| 390 | Register Reg = getMax32BitSubRegister(Reg: CSI[I].getReg(), TRI); |
| 391 | if (Reg > Max) |
| 392 | Max = Reg; |
| 393 | } |
| 394 | return Max; |
| 395 | } |
| 396 | |
| 397 | /// Checks if the basic block contains any instruction that needs a stack |
| 398 | /// frame to be already in place. |
| 399 | static bool needsStackFrame(const MachineBasicBlock &MBB, const BitVector &CSR, |
| 400 | const HexagonRegisterInfo &HRI) { |
| 401 | const MachineFunction *MF = MBB.getParent(); |
| 402 | if (&MBB == &MF->front() && MF->getInfo<HexagonMachineFunctionInfo>() |
| 403 | ->getStackAlignBaseReg() |
| 404 | .isValid()) |
| 405 | return true; |
| 406 | |
| 407 | for (const MachineInstr &MI : MBB) { |
| 408 | if (MI.isCall()) |
| 409 | return true; |
| 410 | unsigned Opc = MI.getOpcode(); |
| 411 | switch (Opc) { |
| 412 | case Hexagon::PS_alloca: |
| 413 | return true; |
| 414 | default: |
| 415 | break; |
| 416 | } |
| 417 | // Check individual operands. |
| 418 | for (const MachineOperand &MO : MI.operands()) { |
| 419 | // While the presence of a frame index does not prove that a stack |
| 420 | // frame will be required, all frame indexes should be within alloc- |
| 421 | // frame/deallocframe. Otherwise, the code that translates a frame |
| 422 | // index into an offset would have to be aware of the placement of |
| 423 | // the frame creation/destruction instructions. |
| 424 | if (MO.isFI()) |
| 425 | return true; |
| 426 | if (MO.isReg()) { |
| 427 | Register R = MO.getReg(); |
| 428 | // Debug instructions may refer to $noreg. |
| 429 | if (!R) |
| 430 | continue; |
| 431 | // Virtual registers will need scavenging, which then may require |
| 432 | // a stack slot. |
| 433 | if (R.isVirtual()) |
| 434 | return true; |
| 435 | for (MCPhysReg S : HRI.subregs_inclusive(Reg: R)) |
| 436 | if (CSR[S]) |
| 437 | return true; |
| 438 | continue; |
| 439 | } |
| 440 | if (MO.isRegMask()) { |
| 441 | // A regmask would normally have all callee-saved registers marked |
| 442 | // as preserved, so this check would not be needed, but in case of |
| 443 | // ever having other regmasks (for other calling conventions), |
| 444 | // make sure they would be processed correctly. |
| 445 | const uint32_t *BM = MO.getRegMask(); |
| 446 | for (int x = CSR.find_first(); x >= 0; x = CSR.find_next(Prev: x)) { |
| 447 | unsigned R = x; |
| 448 | // If this regmask does not preserve a CSR, a frame will be needed. |
| 449 | if (!(BM[R / 32] & (1u << (R % 32)))) |
| 450 | return true; |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | } |
| 455 | return false; |
| 456 | } |
| 457 | |
| 458 | /// Returns true if MBB has a machine instructions that indicates a tail call |
| 459 | /// in the block. |
| 460 | static bool hasTailCall(const MachineBasicBlock &MBB) { |
| 461 | MachineBasicBlock::const_iterator I = MBB.getLastNonDebugInstr(); |
| 462 | if (I == MBB.end()) |
| 463 | return false; |
| 464 | unsigned RetOpc = I->getOpcode(); |
| 465 | return RetOpc == Hexagon::PS_tailcall_i || RetOpc == Hexagon::PS_tailcall_r; |
| 466 | } |
| 467 | |
| 468 | /// Returns true if MBB contains an instruction that returns. |
| 469 | static bool hasReturn(const MachineBasicBlock &MBB) { |
| 470 | for (const MachineInstr &MI : MBB.terminators()) |
| 471 | if (MI.isReturn()) |
| 472 | return true; |
| 473 | return false; |
| 474 | } |
| 475 | |
| 476 | /// Returns the "return" instruction from this block, or nullptr if there |
| 477 | /// isn't any. |
| 478 | static MachineInstr *getReturn(MachineBasicBlock &MBB) { |
| 479 | for (auto &I : MBB) |
| 480 | if (I.isReturn()) |
| 481 | return &I; |
| 482 | return nullptr; |
| 483 | } |
| 484 | |
| 485 | static bool isRestoreCall(unsigned Opc) { |
| 486 | switch (Opc) { |
| 487 | case Hexagon::RESTORE_DEALLOC_RET_JMP_V4: |
| 488 | case Hexagon::RESTORE_DEALLOC_RET_JMP_V4_PIC: |
| 489 | case Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT: |
| 490 | case Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT_PIC: |
| 491 | case Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT: |
| 492 | case Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT_PIC: |
| 493 | case Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4: |
| 494 | case Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_PIC: |
| 495 | return true; |
| 496 | } |
| 497 | return false; |
| 498 | } |
| 499 | |
| 500 | static inline bool isOptNone(const MachineFunction &MF) { |
| 501 | return MF.getFunction().hasOptNone() || |
| 502 | MF.getTarget().getOptLevel() == CodeGenOptLevel::None; |
| 503 | } |
| 504 | |
| 505 | static inline bool isOptSize(const MachineFunction &MF) { |
| 506 | const Function &F = MF.getFunction(); |
| 507 | return F.hasOptSize() && !F.hasMinSize(); |
| 508 | } |
| 509 | |
| 510 | static inline bool isMinSize(const MachineFunction &MF) { |
| 511 | return MF.getFunction().hasMinSize(); |
| 512 | } |
| 513 | |
| 514 | /// Implements shrink-wrapping of the stack frame. By default, stack frame |
| 515 | /// is created in the function entry block, and is cleaned up in every block |
| 516 | /// that returns. This function finds alternate blocks: one for the frame |
| 517 | /// setup (prolog) and one for the cleanup (epilog). |
| 518 | void HexagonFrameLowering::findShrunkPrologEpilog(MachineFunction &MF, |
| 519 | MachineBasicBlock *&PrologB, MachineBasicBlock *&EpilogB) const { |
| 520 | static unsigned ShrinkCounter = 0; |
| 521 | |
| 522 | if (MF.getSubtarget<HexagonSubtarget>().isEnvironmentMusl() && |
| 523 | MF.getFunction().isVarArg()) |
| 524 | return; |
| 525 | // The caller's VSP is saved through R7, which is only known to be free |
| 526 | // in the entry block. |
| 527 | if (MF.getSubtarget<HexagonSubtarget>().useHLXOps()) |
| 528 | return; |
| 529 | if (ShrinkLimit.getPosition()) { |
| 530 | if (ShrinkCounter >= ShrinkLimit) |
| 531 | return; |
| 532 | ShrinkCounter++; |
| 533 | } |
| 534 | |
| 535 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 536 | |
| 537 | MachineDominatorTree MDT; |
| 538 | MDT.recalculate(Func&: MF); |
| 539 | MachinePostDominatorTree MPT; |
| 540 | MPT.recalculate(Func&: MF); |
| 541 | |
| 542 | using UnsignedMap = DenseMap<unsigned, unsigned>; |
| 543 | using RPOTType = ReversePostOrderTraversal<const MachineFunction *>; |
| 544 | |
| 545 | UnsignedMap RPO; |
| 546 | RPOTType RPOT(&MF); |
| 547 | unsigned RPON = 0; |
| 548 | for (auto &I : RPOT) |
| 549 | RPO[I->getNumber()] = RPON++; |
| 550 | |
| 551 | // Don't process functions that have loops, at least for now. Placement |
| 552 | // of prolog and epilog must take loop structure into account. For simpli- |
| 553 | // city don't do it right now. |
| 554 | for (auto &I : MF) { |
| 555 | unsigned BN = RPO[I.getNumber()]; |
| 556 | for (MachineBasicBlock *Succ : I.successors()) |
| 557 | // If found a back-edge, return. |
| 558 | if (RPO[Succ->getNumber()] <= BN) |
| 559 | return; |
| 560 | } |
| 561 | |
| 562 | // Collect the set of blocks that need a stack frame to execute. Scan |
| 563 | // each block for uses/defs of callee-saved registers, calls, etc. |
| 564 | SmallVector<MachineBasicBlock*,16> SFBlocks; |
| 565 | BitVector CSR(Hexagon::NUM_TARGET_REGS); |
| 566 | for (const MCPhysReg *P = HRI.getCalleeSavedRegs(MF: &MF); *P; ++P) |
| 567 | for (MCPhysReg S : HRI.subregs_inclusive(Reg: *P)) |
| 568 | CSR[S] = true; |
| 569 | |
| 570 | for (auto &I : MF) |
| 571 | if (needsStackFrame(MBB: I, CSR, HRI)) |
| 572 | SFBlocks.push_back(Elt: &I); |
| 573 | |
| 574 | LLVM_DEBUG({ |
| 575 | dbgs() << "Blocks needing SF: {" ; |
| 576 | for (auto &B : SFBlocks) |
| 577 | dbgs() << " " << printMBBReference(*B); |
| 578 | dbgs() << " }\n" ; |
| 579 | }); |
| 580 | // No frame needed? |
| 581 | if (SFBlocks.empty()) |
| 582 | return; |
| 583 | |
| 584 | // Pick a common dominator and a common post-dominator. |
| 585 | MachineBasicBlock *DomB = SFBlocks[0]; |
| 586 | for (unsigned i = 1, n = SFBlocks.size(); i < n; ++i) { |
| 587 | DomB = MDT.findNearestCommonDominator(A: DomB, B: SFBlocks[i]); |
| 588 | if (!DomB) |
| 589 | break; |
| 590 | } |
| 591 | MachineBasicBlock *PDomB = SFBlocks[0]; |
| 592 | for (unsigned i = 1, n = SFBlocks.size(); i < n; ++i) { |
| 593 | PDomB = MPT.findNearestCommonDominator(A: PDomB, B: SFBlocks[i]); |
| 594 | if (!PDomB) |
| 595 | break; |
| 596 | } |
| 597 | LLVM_DEBUG({ |
| 598 | dbgs() << "Computed dom block: " ; |
| 599 | if (DomB) |
| 600 | dbgs() << printMBBReference(*DomB); |
| 601 | else |
| 602 | dbgs() << "<null>" ; |
| 603 | dbgs() << ", computed pdom block: " ; |
| 604 | if (PDomB) |
| 605 | dbgs() << printMBBReference(*PDomB); |
| 606 | else |
| 607 | dbgs() << "<null>" ; |
| 608 | dbgs() << "\n" ; |
| 609 | }); |
| 610 | if (!DomB || !PDomB) |
| 611 | return; |
| 612 | |
| 613 | // Make sure that DomB dominates PDomB and PDomB post-dominates DomB. |
| 614 | if (!MDT.dominates(A: DomB, B: PDomB)) { |
| 615 | LLVM_DEBUG(dbgs() << "Dom block does not dominate pdom block\n" ); |
| 616 | return; |
| 617 | } |
| 618 | if (!MPT.dominates(A: PDomB, B: DomB)) { |
| 619 | LLVM_DEBUG(dbgs() << "PDom block does not post-dominate dom block\n" ); |
| 620 | return; |
| 621 | } |
| 622 | |
| 623 | // Finally, everything seems right. |
| 624 | PrologB = DomB; |
| 625 | EpilogB = PDomB; |
| 626 | } |
| 627 | |
| 628 | /// Perform most of the PEI work here: |
| 629 | /// - saving/restoring of the callee-saved registers, |
| 630 | /// - stack frame creation and destruction. |
| 631 | /// Normally, this work is distributed among various functions, but doing it |
| 632 | /// in one place allows shrink-wrapping of the stack frame. |
| 633 | void HexagonFrameLowering::emitPrologue(MachineFunction &MF, |
| 634 | MachineBasicBlock &MBB) const { |
| 635 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 636 | |
| 637 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 638 | const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
| 639 | |
| 640 | MachineBasicBlock *PrologB = &MF.front(), *EpilogB = nullptr; |
| 641 | if (EnableShrinkWrapping) |
| 642 | findShrunkPrologEpilog(MF, PrologB, EpilogB); |
| 643 | |
| 644 | bool PrologueStubs = false; |
| 645 | MachineBasicBlock::iterator AfterCSR = |
| 646 | insertCSRSpillsInBlock(MBB&: *PrologB, CSI, HRI, PrologueStubs); |
| 647 | // Insert PS_aligna after all CSR spills. |
| 648 | // PS_aligna initializes the AP register with an aligned |
| 649 | // value derived from FP. Since AP is a callee-saved register, its original |
| 650 | // value must be saved before it is overwritten, and it must be defined |
| 651 | // before any AP-relative stack accesses. |
| 652 | insertAlignaInBlock(MBB&: *PrologB, InsertPt: AfterCSR); |
| 653 | insertPrologueInBlock(MBB&: *PrologB, PrologueStubs); |
| 654 | // Insert the SCS prologue after all FrameSetup instructions so that it |
| 655 | // follows allocframe and any CSR spills in the instruction stream. The |
| 656 | // packetizer may still fuse the SCS store with the first call in the |
| 657 | // function, but because Hexagon packets use old-value reads the original |
| 658 | // R31 is always what is stored. |
| 659 | { |
| 660 | MachineBasicBlock::iterator AfterProlog = PrologB->begin(); |
| 661 | while (AfterProlog != PrologB->end() && |
| 662 | AfterProlog->getFlag(Flag: MachineInstr::FrameSetup)) |
| 663 | ++AfterProlog; |
| 664 | DebugLoc PrologDL = PrologB->findDebugLoc(MBBI: AfterProlog); |
| 665 | emitSCSPrologue(MF, MBB&: *PrologB, MI: AfterProlog, DL: PrologDL); |
| 666 | } |
| 667 | updateEntryPaths(MF, SaveB&: *PrologB); |
| 668 | |
| 669 | if (EpilogB) { |
| 670 | insertCSRRestoresInBlock(MBB&: *EpilogB, CSI, HRI); |
| 671 | insertEpilogueInBlock(MBB&: *EpilogB); |
| 672 | } else { |
| 673 | for (auto &B : MF) |
| 674 | if (B.isReturnBlock()) |
| 675 | insertCSRRestoresInBlock(MBB&: B, CSI, HRI); |
| 676 | |
| 677 | for (auto &B : MF) |
| 678 | if (B.isReturnBlock()) |
| 679 | insertEpilogueInBlock(MBB&: B); |
| 680 | |
| 681 | for (auto &B : MF) { |
| 682 | if (B.empty()) |
| 683 | continue; |
| 684 | MachineInstr *RetI = getReturn(MBB&: B); |
| 685 | if (!RetI || isRestoreCall(Opc: RetI->getOpcode())) |
| 686 | continue; |
| 687 | for (auto &R : CSI) |
| 688 | RetI->addOperand(Op: MachineOperand::CreateReg(Reg: R.getReg(), isDef: false, isImp: true)); |
| 689 | } |
| 690 | } |
| 691 | |
| 692 | if (EpilogB) { |
| 693 | // If there is an epilog block, it may not have a return instruction. |
| 694 | // In such case, we need to add the callee-saved registers as live-ins |
| 695 | // in all blocks on all paths from the epilog to any return block. |
| 696 | unsigned MaxBN = MF.getNumBlockIDs(); |
| 697 | BitVector DoneT(MaxBN+1), DoneF(MaxBN+1), Path(MaxBN+1); |
| 698 | updateExitPaths(MBB&: *EpilogB, RestoreB&: *EpilogB, DoneT, DoneF, Path); |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | /// Returns true if the target can safely skip saving callee-saved registers |
| 703 | /// for noreturn nounwind functions. |
| 704 | bool HexagonFrameLowering::enableCalleeSaveSkip( |
| 705 | const MachineFunction &MF) const { |
| 706 | const auto &F = MF.getFunction(); |
| 707 | assert(F.hasFnAttribute(Attribute::NoReturn) && |
| 708 | F.getFunction().hasFnAttribute(Attribute::NoUnwind) && |
| 709 | !F.getFunction().hasFnAttribute(Attribute::UWTable)); |
| 710 | (void)F; |
| 711 | |
| 712 | // No need to save callee saved registers if the function does not return. |
| 713 | return MF.getSubtarget<HexagonSubtarget>().noreturnStackElim(); |
| 714 | } |
| 715 | |
| 716 | // Helper function used to determine when to eliminate the stack frame for |
| 717 | // functions marked as noreturn and when the noreturn-stack-elim options are |
| 718 | // specified. When both these conditions are true, then a FP may not be needed |
| 719 | // if the function makes a call. It is very similar to enableCalleeSaveSkip, |
| 720 | // but it used to check if the allocframe can be eliminated as well. |
| 721 | static bool enableAllocFrameElim(const MachineFunction &MF) { |
| 722 | const auto &F = MF.getFunction(); |
| 723 | const auto &MFI = MF.getFrameInfo(); |
| 724 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 725 | assert(!MFI.hasVarSizedObjects() && |
| 726 | !HST.getRegisterInfo()->hasStackRealignment(MF)); |
| 727 | return F.hasFnAttribute(Kind: Attribute::NoReturn) && |
| 728 | F.hasFnAttribute(Kind: Attribute::NoUnwind) && |
| 729 | !F.hasFnAttribute(Kind: Attribute::UWTable) && HST.noreturnStackElim() && |
| 730 | MFI.getStackSize() == 0; |
| 731 | } |
| 732 | |
| 733 | void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB, |
| 734 | bool PrologueStubs) const { |
| 735 | MachineFunction &MF = *MBB.getParent(); |
| 736 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 737 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 738 | auto &HII = *HST.getInstrInfo(); |
| 739 | auto &HRI = *HST.getRegisterInfo(); |
| 740 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 741 | |
| 742 | Align MaxAlign = std::max(a: getMaxStackAlign(MF), b: getStackAlign()); |
| 743 | uint64_t VTCMFrameSize = HMFI.getVTCMFrameInfo().getStackSize(); |
| 744 | |
| 745 | // Calculate the total stack frame size. |
| 746 | // Get the number of bytes to allocate from the FrameInfo. |
| 747 | unsigned FrameSize = MFI.getStackSize(); |
| 748 | // Round up the max call frame size to the max alignment on the stack. |
| 749 | unsigned MaxCFA = alignTo(Size: MFI.getMaxCallFrameSize(), A: MaxAlign); |
| 750 | MFI.setMaxCallFrameSize(MaxCFA); |
| 751 | |
| 752 | FrameSize = MaxCFA + alignTo(Size: FrameSize, A: MaxAlign); |
| 753 | MFI.setStackSize(FrameSize); |
| 754 | |
| 755 | bool AlignStack = (MaxAlign > getStackAlign()); |
| 756 | |
| 757 | // Get the number of bytes to allocate from the FrameInfo. |
| 758 | unsigned NumBytes = MFI.getStackSize(); |
| 759 | Register SP = HRI.getStackRegister(); |
| 760 | unsigned MaxCF = MFI.getMaxCallFrameSize(); |
| 761 | MachineBasicBlock::iterator InsertPt = MBB.begin(); |
| 762 | |
| 763 | SmallVector<MachineInstr *, 4> AdjustRegs; |
| 764 | for (auto &MBB : MF) |
| 765 | for (auto &MI : MBB) |
| 766 | if (MI.getOpcode() == Hexagon::PS_alloca) |
| 767 | AdjustRegs.push_back(Elt: &MI); |
| 768 | |
| 769 | for (auto *MI : AdjustRegs) { |
| 770 | assert((MI->getOpcode() == Hexagon::PS_alloca) && "Expected alloca" ); |
| 771 | expandAlloca(AI: MI, MF, TII: HII, SP, CF: MaxCF); |
| 772 | MI->eraseFromParent(); |
| 773 | } |
| 774 | |
| 775 | DebugLoc dl = MBB.findDebugLoc(MBBI: InsertPt); |
| 776 | |
| 777 | if (MF.getFunction().isVarArg() && |
| 778 | MF.getSubtarget<HexagonSubtarget>().isEnvironmentMusl()) { |
| 779 | // Calculate the size of register saved area. |
| 780 | int NumVarArgRegs = 6 - FirstVarArgSavedReg; |
| 781 | int RegisterSavedAreaSizePlusPadding = (NumVarArgRegs % 2 == 0) |
| 782 | ? NumVarArgRegs * 4 |
| 783 | : NumVarArgRegs * 4 + 4; |
| 784 | if (RegisterSavedAreaSizePlusPadding > 0) { |
| 785 | // Decrement the stack pointer by size of register saved area plus |
| 786 | // padding if any. |
| 787 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SP) |
| 788 | .addReg(RegNo: SP) |
| 789 | .addImm(Val: -RegisterSavedAreaSizePlusPadding) |
| 790 | .setMIFlag(MachineInstr::FrameSetup); |
| 791 | |
| 792 | int NumBytes = 0; |
| 793 | // Copy all the named arguments below register saved area. |
| 794 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 795 | for (int i = HMFI.getFirstNamedArgFrameIndex(), |
| 796 | e = HMFI.getLastNamedArgFrameIndex(); i >= e; --i) { |
| 797 | // HLX arguments are on the VTCM stack. |
| 798 | if (MFI.getStackID(ObjectIdx: i) == TargetStackID::NoAlloc) |
| 799 | continue; |
| 800 | uint64_t ObjSize = MFI.getObjectSize(ObjectIdx: i); |
| 801 | Align ObjAlign = MFI.getObjectAlign(ObjectIdx: i); |
| 802 | |
| 803 | // Determine the kind of load/store that should be used. |
| 804 | unsigned LDOpc, STOpc; |
| 805 | uint64_t OpcodeChecker = ObjAlign.value(); |
| 806 | |
| 807 | // Handle cases where alignment of an object is > its size. |
| 808 | if (ObjAlign > ObjSize) { |
| 809 | if (ObjSize <= 1) |
| 810 | OpcodeChecker = 1; |
| 811 | else if (ObjSize <= 2) |
| 812 | OpcodeChecker = 2; |
| 813 | else if (ObjSize <= 4) |
| 814 | OpcodeChecker = 4; |
| 815 | else if (ObjSize > 4) |
| 816 | OpcodeChecker = 8; |
| 817 | } |
| 818 | |
| 819 | switch (OpcodeChecker) { |
| 820 | case 1: |
| 821 | LDOpc = Hexagon::L2_loadrb_io; |
| 822 | STOpc = Hexagon::S2_storerb_io; |
| 823 | break; |
| 824 | case 2: |
| 825 | LDOpc = Hexagon::L2_loadrh_io; |
| 826 | STOpc = Hexagon::S2_storerh_io; |
| 827 | break; |
| 828 | case 4: |
| 829 | LDOpc = Hexagon::L2_loadri_io; |
| 830 | STOpc = Hexagon::S2_storeri_io; |
| 831 | break; |
| 832 | case 8: |
| 833 | default: |
| 834 | LDOpc = Hexagon::L2_loadrd_io; |
| 835 | STOpc = Hexagon::S2_storerd_io; |
| 836 | break; |
| 837 | } |
| 838 | |
| 839 | Register RegUsed = LDOpc == Hexagon::L2_loadrd_io ? Hexagon::D3 |
| 840 | : Hexagon::R6; |
| 841 | int LoadStoreCount = ObjSize / OpcodeChecker; |
| 842 | |
| 843 | if (ObjSize % OpcodeChecker) |
| 844 | ++LoadStoreCount; |
| 845 | |
| 846 | // Get the start location of the load. NumBytes is basically the |
| 847 | // offset from the stack pointer of previous function, which would be |
| 848 | // the caller in this case, as this function has variable argument |
| 849 | // list. |
| 850 | if (NumBytes != 0) |
| 851 | NumBytes = alignTo(Size: NumBytes, A: ObjAlign); |
| 852 | |
| 853 | int Count = 0; |
| 854 | while (Count < LoadStoreCount) { |
| 855 | // Load the value of the named argument on stack. |
| 856 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: LDOpc), DestReg: RegUsed) |
| 857 | .addReg(RegNo: SP) |
| 858 | .addImm(Val: RegisterSavedAreaSizePlusPadding + |
| 859 | ObjAlign.value() * Count + NumBytes) |
| 860 | .setMIFlag(MachineInstr::FrameSetup); |
| 861 | |
| 862 | // Store it below the register saved area plus padding. |
| 863 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: STOpc)) |
| 864 | .addReg(RegNo: SP) |
| 865 | .addImm(Val: ObjAlign.value() * Count + NumBytes) |
| 866 | .addReg(RegNo: RegUsed) |
| 867 | .setMIFlag(MachineInstr::FrameSetup); |
| 868 | |
| 869 | Count++; |
| 870 | } |
| 871 | NumBytes += MFI.getObjectSize(ObjectIdx: i); |
| 872 | } |
| 873 | |
| 874 | // Make NumBytes 8 byte aligned |
| 875 | NumBytes = alignTo(Value: NumBytes, Align: 8); |
| 876 | |
| 877 | // If the number of registers having variable arguments is odd, |
| 878 | // leave 4 bytes of padding to get to the location where first |
| 879 | // variable argument which was passed through register was copied. |
| 880 | NumBytes = (NumVarArgRegs % 2 == 0) ? NumBytes : NumBytes + 4; |
| 881 | |
| 882 | for (int j = FirstVarArgSavedReg, i = 0; j < 6; ++j, ++i) { |
| 883 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::S2_storeri_io)) |
| 884 | .addReg(RegNo: SP) |
| 885 | .addImm(Val: NumBytes + 4 * i) |
| 886 | .addReg(RegNo: Hexagon::R0 + j) |
| 887 | .setMIFlag(MachineInstr::FrameSetup); |
| 888 | } |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | if (hasFP(MF)) { |
| 893 | insertAllocframe(MBB, InsertPt, NumBytes); |
| 894 | if (AlignStack) { |
| 895 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_andir), DestReg: SP) |
| 896 | .addReg(RegNo: SP) |
| 897 | .addImm(Val: -int64_t(MaxAlign.value())); |
| 898 | } |
| 899 | // If the stack-checking is enabled, and we spilled the callee-saved |
| 900 | // registers inline (i.e. did not use a spill function), then call |
| 901 | // the stack checker directly. |
| 902 | if (EnableStackOVFSanitizer && !PrologueStubs) |
| 903 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::PS_call_stk)) |
| 904 | .addExternalSymbol(FnName: "__runtime_stack_check" ); |
| 905 | } else if (NumBytes > 0) { |
| 906 | assert(alignTo(NumBytes, 8) == NumBytes); |
| 907 | auto *TLI = HST.getTargetLowering(); |
| 908 | bool NeedsProbing = TLI->hasInlineStackProbe(MF); |
| 909 | unsigned ProbeSize = 0; |
| 910 | if (NeedsProbing) { |
| 911 | Align StackAlign = getStackAlign(); |
| 912 | ProbeSize = TLI->getStackProbeSize(MF, StackAlign); |
| 913 | } |
| 914 | if (NeedsProbing && NumBytes > ProbeSize) { |
| 915 | // Compute target SP in R28 (caller-saved scratch). |
| 916 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: Hexagon::R28) |
| 917 | .addReg(RegNo: SP) |
| 918 | .addImm(Val: -int(NumBytes)) |
| 919 | .setMIFlag(MachineInstr::FrameSetup); |
| 920 | // Emit pseudo to be expanded by inlineStackProbe(). |
| 921 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::PS_probed_stackalloc)) |
| 922 | .addReg(RegNo: Hexagon::R28) |
| 923 | .setMIFlag(MachineInstr::FrameSetup); |
| 924 | } else { |
| 925 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SP) |
| 926 | .addReg(RegNo: SP) |
| 927 | .addImm(Val: -int(NumBytes)) |
| 928 | .setMIFlag(MachineInstr::FrameSetup); |
| 929 | } |
| 930 | } |
| 931 | if (!VTCMFrameSize) |
| 932 | return; |
| 933 | if (int VSPFI = HMFI.getVSPFrameIndex(); VSPFI >= 0) { |
| 934 | // Save the caller's VSP before allocating the vector frame. R7 is not |
| 935 | // an argument register, and the prologue is in the entry block (see |
| 936 | // findShrunkPrologEpilog). |
| 937 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_tfrcrr), DestReg: Hexagon::R7) |
| 938 | .addReg(RegNo: Hexagon::VSP) |
| 939 | .setMIFlag(MachineInstr::FrameSetup); |
| 940 | HII.storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: Hexagon::R7, /*isKill=*/true, FrameIndex: VSPFI, |
| 941 | RC: &Hexagon::IntRegsRegClass, VReg: Register()); |
| 942 | std::prev(x: InsertPt)->setFlag(MachineInstr::FrameSetup); |
| 943 | } |
| 944 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::V2_vallocframe_imm)) |
| 945 | .addImm(Val: VTCMFrameSize / 2048) |
| 946 | .setMIFlag(MachineInstr::FrameSetup); |
| 947 | } |
| 948 | |
| 949 | void HexagonFrameLowering::insertEpilogueInBlock(MachineBasicBlock &MBB) const { |
| 950 | MachineFunction &MF = *MBB.getParent(); |
| 951 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 952 | auto &HII = *HST.getInstrInfo(); |
| 953 | auto &HRI = *HST.getRegisterInfo(); |
| 954 | Register SP = HRI.getStackRegister(); |
| 955 | |
| 956 | MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator(); |
| 957 | DebugLoc dl = MBB.findDebugLoc(MBBI: InsertPt); |
| 958 | |
| 959 | const auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 960 | uint64_t VTCMFrameSize = HMFI.getVTCMFrameInfo().getStackSize(); |
| 961 | if (VTCMFrameSize) |
| 962 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::V2_vdeallocframe_imm)) |
| 963 | .addImm(Val: VTCMFrameSize / 2048); |
| 964 | |
| 965 | if (!hasFP(MF)) { |
| 966 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 967 | unsigned NumBytes = MFI.getStackSize(); |
| 968 | if (MF.getFunction().isVarArg() && |
| 969 | MF.getSubtarget<HexagonSubtarget>().isEnvironmentMusl()) { |
| 970 | // On Hexagon Linux, deallocate the stack for the register saved area. |
| 971 | int NumVarArgRegs = 6 - FirstVarArgSavedReg; |
| 972 | int RegisterSavedAreaSizePlusPadding = (NumVarArgRegs % 2 == 0) ? |
| 973 | (NumVarArgRegs * 4) : (NumVarArgRegs * 4 + 4); |
| 974 | NumBytes += RegisterSavedAreaSizePlusPadding; |
| 975 | } |
| 976 | if (NumBytes) { |
| 977 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SP) |
| 978 | .addReg(RegNo: SP) |
| 979 | .addImm(Val: NumBytes); |
| 980 | } |
| 981 | return; |
| 982 | } |
| 983 | |
| 984 | MachineInstr *RetI = getReturn(MBB); |
| 985 | unsigned RetOpc = RetI ? RetI->getOpcode() : 0; |
| 986 | |
| 987 | // Handle EH_RETURN. |
| 988 | if (RetOpc == Hexagon::EH_RETURN_JMPR) { |
| 989 | // EH paths overwrite R31 with a handler address; the shadow stack is |
| 990 | // not read on this path, so no SCS epilogue is needed. |
| 991 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::L2_deallocframe)) |
| 992 | .addDef(RegNo: Hexagon::D15) |
| 993 | .addReg(RegNo: Hexagon::R30); |
| 994 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_add), DestReg: SP) |
| 995 | .addReg(RegNo: SP) |
| 996 | .addReg(RegNo: Hexagon::R28); |
| 997 | return; |
| 998 | } |
| 999 | |
| 1000 | // Check for RESTORE_DEALLOC_RET* tail call. Don't emit an extra dealloc- |
| 1001 | // frame instruction if we encounter it. |
| 1002 | // These are restore stubs, which useRestoreFunction() never selects when SCS |
| 1003 | // is active (they do deallocframe+jumpr, bypassing the SCS epilogue), so no |
| 1004 | // SCS epilogue is needed here. |
| 1005 | if (RetOpc == Hexagon::RESTORE_DEALLOC_RET_JMP_V4 || |
| 1006 | RetOpc == Hexagon::RESTORE_DEALLOC_RET_JMP_V4_PIC || |
| 1007 | RetOpc == Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT || |
| 1008 | RetOpc == Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT_PIC) { |
| 1009 | MachineBasicBlock::iterator It = RetI; |
| 1010 | ++It; |
| 1011 | // Delete all instructions after the RESTORE (except labels). |
| 1012 | while (It != MBB.end()) { |
| 1013 | if (!It->isLabel()) |
| 1014 | It = MBB.erase(I: It); |
| 1015 | else |
| 1016 | ++It; |
| 1017 | } |
| 1018 | return; |
| 1019 | } |
| 1020 | |
| 1021 | // It is possible that the restoring code is a call to a library function. |
| 1022 | // All of the restore* functions include "deallocframe", so we need to make |
| 1023 | // sure that we don't add an extra one. |
| 1024 | bool NeedsSCS = MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack); |
| 1025 | bool NeedsDeallocframe = true; |
| 1026 | unsigned PrevOpc = 0; |
| 1027 | if (!MBB.empty() && InsertPt != MBB.begin()) { |
| 1028 | MachineBasicBlock::iterator PrevIt = std::prev(x: InsertPt); |
| 1029 | PrevOpc = PrevIt->getOpcode(); |
| 1030 | if (PrevOpc == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4 || |
| 1031 | PrevOpc == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_PIC || |
| 1032 | PrevOpc == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT || |
| 1033 | PrevOpc == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT_PIC || |
| 1034 | PrevOpc == Hexagon::PS_call_nr || PrevOpc == Hexagon::PS_callr_nr) |
| 1035 | NeedsDeallocframe = false; |
| 1036 | } |
| 1037 | |
| 1038 | if (!MF.getSubtarget<HexagonSubtarget>().isEnvironmentMusl() || |
| 1039 | !MF.getFunction().isVarArg()) { |
| 1040 | if (!NeedsDeallocframe) { |
| 1041 | // RESTORE_DEALLOC_BEFORE_TAILCALL is a restore stub, which |
| 1042 | // useRestoreFunction() never selects when SCS is active. |
| 1043 | // PS_call_nr/PS_callr_nr are noreturn calls so the shadow stack entry |
| 1044 | // is never read - no SCS epilogue is needed on either path. |
| 1045 | if (NeedsSCS && PrevOpc != Hexagon::PS_call_nr && |
| 1046 | PrevOpc != Hexagon::PS_callr_nr) |
| 1047 | report_fatal_error(reason: "SCS with RESTORE_DEALLOC stub: " |
| 1048 | "useRestoreFunction() should have prevented this" ); |
| 1049 | return; |
| 1050 | } |
| 1051 | // If the returning instruction is PS_jmpret, replace it with |
| 1052 | // dealloc_return, otherwise just add deallocframe. The function |
| 1053 | // could be returning via a tail call. |
| 1054 | if (RetOpc != Hexagon::PS_jmpret || DisableDeallocRet || NeedsSCS) { |
| 1055 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::L2_deallocframe)) |
| 1056 | .addDef(RegNo: Hexagon::D15) |
| 1057 | .addReg(RegNo: Hexagon::R30); |
| 1058 | // When shadow call stack is active, overwrite R31 restored by |
| 1059 | // deallocframe with the shadow-stack copy, then retract the pointer. |
| 1060 | if (NeedsSCS) |
| 1061 | emitSCSEpilogue(MF, MBB, MI: InsertPt, DL: dl); |
| 1062 | return; |
| 1063 | } |
| 1064 | unsigned NewOpc = Hexagon::L4_return; |
| 1065 | MachineInstrBuilder NewI = BuildMI(BB&: MBB, I: RetI, MIMD: dl, MCID: HII.get(Opcode: NewOpc)) |
| 1066 | .addDef(RegNo: Hexagon::D15) |
| 1067 | .addReg(RegNo: Hexagon::R30); |
| 1068 | // Avoid duplicating the pc implicit-def. |
| 1069 | for (const MachineOperand &MO : drop_begin(RangeOrContainer: RetI->implicit_operands())) |
| 1070 | NewI.add(MO); |
| 1071 | MBB.erase(I: RetI); |
| 1072 | } else { |
| 1073 | // L2_deallocframe instruction after it. |
| 1074 | // Calculate the size of register saved area. |
| 1075 | int NumVarArgRegs = 6 - FirstVarArgSavedReg; |
| 1076 | int RegisterSavedAreaSizePlusPadding = (NumVarArgRegs % 2 == 0) ? |
| 1077 | (NumVarArgRegs * 4) : (NumVarArgRegs * 4 + 4); |
| 1078 | |
| 1079 | MachineBasicBlock::iterator Term = MBB.getFirstTerminator(); |
| 1080 | MachineBasicBlock::iterator I = (Term == MBB.begin()) ? MBB.end() |
| 1081 | : std::prev(x: Term); |
| 1082 | bool HasRestoreStub = |
| 1083 | I != MBB.end() && |
| 1084 | (I->getOpcode() == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT || |
| 1085 | I->getOpcode() == |
| 1086 | Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT_PIC || |
| 1087 | I->getOpcode() == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4 || |
| 1088 | I->getOpcode() == Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_PIC); |
| 1089 | if (!HasRestoreStub) |
| 1090 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::L2_deallocframe)) |
| 1091 | .addDef(RegNo: Hexagon::D15) |
| 1092 | .addReg(RegNo: Hexagon::R30); |
| 1093 | if (RegisterSavedAreaSizePlusPadding != 0) |
| 1094 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SP) |
| 1095 | .addReg(RegNo: SP) |
| 1096 | .addImm(Val: RegisterSavedAreaSizePlusPadding); |
| 1097 | // RESTORE_DEALLOC stubs are never selected when SCS is active (see |
| 1098 | // useRestoreFunction()), so only emit the SCS epilogue when we emitted |
| 1099 | // our own deallocframe above. |
| 1100 | if (NeedsSCS && !HasRestoreStub) |
| 1101 | emitSCSEpilogue(MF, MBB, MI: InsertPt, DL: dl); |
| 1102 | } |
| 1103 | } |
| 1104 | |
| 1105 | void HexagonFrameLowering::insertAllocframe(MachineBasicBlock &MBB, |
| 1106 | MachineBasicBlock::iterator InsertPt, unsigned NumBytes) const { |
| 1107 | MachineFunction &MF = *MBB.getParent(); |
| 1108 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1109 | auto &HII = *HST.getInstrInfo(); |
| 1110 | auto &HRI = *HST.getRegisterInfo(); |
| 1111 | |
| 1112 | // Check for overflow. |
| 1113 | // Hexagon_TODO: Ugh! hardcoding. Is there an API that can be used? |
| 1114 | const unsigned int ALLOCFRAME_MAX = 16384; |
| 1115 | |
| 1116 | // Create a dummy memory operand to avoid allocframe from being treated as |
| 1117 | // a volatile memory reference. |
| 1118 | auto *MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getStack(MF, Offset: 0), |
| 1119 | F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4)); |
| 1120 | |
| 1121 | DebugLoc dl = MBB.findDebugLoc(MBBI: InsertPt); |
| 1122 | Register SP = HRI.getStackRegister(); |
| 1123 | |
| 1124 | auto *TLI = HST.getTargetLowering(); |
| 1125 | bool NeedsProbing = TLI->hasInlineStackProbe(MF) && NumBytes > 0; |
| 1126 | unsigned ProbeSize = 0; |
| 1127 | if (NeedsProbing) { |
| 1128 | Align StackAlign = getStackAlign(); |
| 1129 | ProbeSize = TLI->getStackProbeSize(MF, StackAlign); |
| 1130 | } |
| 1131 | |
| 1132 | if (NeedsProbing && NumBytes > ProbeSize) { |
| 1133 | // Emit allocframe(#0) to save FP/LR only. |
| 1134 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::S2_allocframe)) |
| 1135 | .addDef(RegNo: SP) |
| 1136 | .addReg(RegNo: SP) |
| 1137 | .addImm(Val: 0) |
| 1138 | .addMemOperand(MMO) |
| 1139 | .setMIFlag(MachineInstr::FrameSetup); |
| 1140 | |
| 1141 | // Compute target SP in R28 (caller-saved scratch). |
| 1142 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: Hexagon::R28) |
| 1143 | .addReg(RegNo: SP) |
| 1144 | .addImm(Val: -int(NumBytes)) |
| 1145 | .setMIFlag(MachineInstr::FrameSetup); |
| 1146 | |
| 1147 | // Emit pseudo to be expanded by inlineStackProbe(). |
| 1148 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::PS_probed_stackalloc)) |
| 1149 | .addReg(RegNo: Hexagon::R28) |
| 1150 | .setMIFlag(MachineInstr::FrameSetup); |
| 1151 | } else if (NumBytes >= ALLOCFRAME_MAX) { |
| 1152 | // Emit allocframe(#0). |
| 1153 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::S2_allocframe)) |
| 1154 | .addDef(RegNo: SP) |
| 1155 | .addReg(RegNo: SP) |
| 1156 | .addImm(Val: 0) |
| 1157 | .addMemOperand(MMO) |
| 1158 | .setMIFlag(MachineInstr::FrameSetup); |
| 1159 | |
| 1160 | // Subtract the size from the stack pointer. |
| 1161 | Register SP = HRI.getStackRegister(); |
| 1162 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: SP) |
| 1163 | .addReg(RegNo: SP) |
| 1164 | .addImm(Val: -int(NumBytes)) |
| 1165 | .setMIFlag(MachineInstr::FrameSetup); |
| 1166 | } else { |
| 1167 | BuildMI(BB&: MBB, I: InsertPt, MIMD: dl, MCID: HII.get(Opcode: Hexagon::S2_allocframe)) |
| 1168 | .addDef(RegNo: SP) |
| 1169 | .addReg(RegNo: SP) |
| 1170 | .addImm(Val: NumBytes) |
| 1171 | .addMemOperand(MMO) |
| 1172 | .setMIFlag(MachineInstr::FrameSetup); |
| 1173 | } |
| 1174 | } |
| 1175 | |
| 1176 | void HexagonFrameLowering::inlineStackProbe( |
| 1177 | MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { |
| 1178 | // Collect PS_probed_stackalloc pseudos to expand. Collecting first avoids |
| 1179 | // issues with modifying the block while iterating. |
| 1180 | SmallVector<MachineInstr *, 2> ToReplace; |
| 1181 | for (MachineInstr &MI : PrologueMBB) |
| 1182 | if (MI.getOpcode() == Hexagon::PS_probed_stackalloc) |
| 1183 | ToReplace.push_back(Elt: &MI); |
| 1184 | |
| 1185 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1186 | auto &HII = *HST.getInstrInfo(); |
| 1187 | auto *TLI = HST.getTargetLowering(); |
| 1188 | Align StackAlign = getStackAlign(); |
| 1189 | unsigned ProbeSize = TLI->getStackProbeSize(MF, StackAlign); |
| 1190 | MachineInstr::MIFlag Flags = MachineInstr::FrameSetup; |
| 1191 | |
| 1192 | for (MachineInstr *MI : ToReplace) { |
| 1193 | MachineBasicBlock::iterator MBBI = MI->getIterator(); |
| 1194 | DebugLoc DL = PrologueMBB.findDebugLoc(MBBI); |
| 1195 | Register TargetReg = MI->getOperand(i: 0).getReg(); |
| 1196 | |
| 1197 | // Split the block: everything after the pseudo goes into ExitMBB. |
| 1198 | MachineBasicBlock *MBB = MI->getParent(); |
| 1199 | MachineFunction::iterator InsertPt = std::next(x: MBB->getIterator()); |
| 1200 | MachineBasicBlock *LoopMBB = |
| 1201 | MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock()); |
| 1202 | MF.insert(MBBI: InsertPt, MBB: LoopMBB); |
| 1203 | MachineBasicBlock *ExitMBB = |
| 1204 | MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock()); |
| 1205 | MF.insert(MBBI: InsertPt, MBB: ExitMBB); |
| 1206 | |
| 1207 | // Move everything after the pseudo into ExitMBB. |
| 1208 | ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end()); |
| 1209 | ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB); |
| 1210 | |
| 1211 | // LoopMBB: probe each page by decrementing SP and storing zero. |
| 1212 | // When NumBytes is not an exact multiple of ProbeSize the loop |
| 1213 | // will overshoot by up to ProbeSize-1 bytes; the final r29 = r28 |
| 1214 | // in ExitMBB corrects SP to the true target. |
| 1215 | // |
| 1216 | // The store is placed before the compare+branch so that the |
| 1217 | // packetizer can bundle them into a single VLIW packet. All |
| 1218 | // non-predicated instructions in a packet commit unconditionally, |
| 1219 | // so the probe store executes on every iteration including the |
| 1220 | // last (when the branch falls through). |
| 1221 | // |
| 1222 | // r29 = add(r29, #-ProbeSize) |
| 1223 | // memw(r29+#0) = #0 |
| 1224 | // p0 = cmp.gtu(r29, r28) |
| 1225 | // if (p0) jump LoopMBB |
| 1226 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), |
| 1227 | DestReg: Hexagon::R29) |
| 1228 | .addReg(RegNo: Hexagon::R29) |
| 1229 | .addImm(Val: -int(ProbeSize)) |
| 1230 | .setMIFlags(Flags); |
| 1231 | |
| 1232 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::S4_storeiri_io)) |
| 1233 | .addReg(RegNo: Hexagon::R29) |
| 1234 | .addImm(Val: 0) |
| 1235 | .addImm(Val: 0) |
| 1236 | .setMIFlags(Flags); |
| 1237 | |
| 1238 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::C2_cmpgtu), |
| 1239 | DestReg: Hexagon::P0) |
| 1240 | .addReg(RegNo: Hexagon::R29) |
| 1241 | .addReg(RegNo: TargetReg) |
| 1242 | .setMIFlags(Flags); |
| 1243 | |
| 1244 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::J2_jumpt)) |
| 1245 | .addReg(RegNo: Hexagon::P0) |
| 1246 | .addMBB(MBB: LoopMBB) |
| 1247 | .setMIFlags(Flags); |
| 1248 | |
| 1249 | // ExitMBB: set final SP. |
| 1250 | BuildMI(BB&: *ExitMBB, I: ExitMBB->begin(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfr), |
| 1251 | DestReg: Hexagon::R29) |
| 1252 | .addReg(RegNo: TargetReg) |
| 1253 | .setMIFlags(Flags); |
| 1254 | |
| 1255 | // Set up CFG edges. |
| 1256 | MBB->addSuccessor(Succ: LoopMBB); |
| 1257 | LoopMBB->addSuccessor(Succ: LoopMBB); |
| 1258 | LoopMBB->addSuccessor(Succ: ExitMBB); |
| 1259 | |
| 1260 | // Remove the pseudo. |
| 1261 | MI->eraseFromParent(); |
| 1262 | |
| 1263 | // Recompute live-ins for the new blocks. |
| 1264 | fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopMBB}); |
| 1265 | } |
| 1266 | } |
| 1267 | |
| 1268 | void HexagonFrameLowering::insertAlignaInBlock( |
| 1269 | MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt) const { |
| 1270 | MachineFunction &MF = *MBB.getParent(); |
| 1271 | Register AP = |
| 1272 | MF.getInfo<HexagonMachineFunctionInfo>()->getStackAlignBaseReg(); |
| 1273 | if (!AP.isValid()) |
| 1274 | return; |
| 1275 | |
| 1276 | assert(needsAligna(MF) && "Unexpected stack align base register" ); |
| 1277 | |
| 1278 | auto &HII = *MF.getSubtarget<HexagonSubtarget>().getInstrInfo(); |
| 1279 | Align MaxAlign = std::max(a: MF.getFrameInfo().getMaxAlign(), b: getStackAlign()); |
| 1280 | DebugLoc DL = MBB.findDebugLoc(MBBI: InsertPt); |
| 1281 | BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: HII.get(Opcode: Hexagon::PS_aligna), DestReg: AP) |
| 1282 | .addImm(Val: MaxAlign.value()); |
| 1283 | } |
| 1284 | |
| 1285 | void HexagonFrameLowering::updateEntryPaths(MachineFunction &MF, |
| 1286 | MachineBasicBlock &SaveB) const { |
| 1287 | SetVector<unsigned> Worklist; |
| 1288 | |
| 1289 | MachineBasicBlock &EntryB = MF.front(); |
| 1290 | Worklist.insert(X: EntryB.getNumber()); |
| 1291 | |
| 1292 | unsigned SaveN = SaveB.getNumber(); |
| 1293 | auto &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
| 1294 | |
| 1295 | for (unsigned i = 0; i < Worklist.size(); ++i) { |
| 1296 | unsigned BN = Worklist[i]; |
| 1297 | MachineBasicBlock &MBB = *MF.getBlockNumbered(N: BN); |
| 1298 | for (auto &R : CSI) |
| 1299 | if (!MBB.isLiveIn(Reg: R.getReg())) |
| 1300 | MBB.addLiveIn(PhysReg: R.getReg()); |
| 1301 | if (BN != SaveN) |
| 1302 | for (auto &SB : MBB.successors()) |
| 1303 | Worklist.insert(X: SB->getNumber()); |
| 1304 | } |
| 1305 | } |
| 1306 | |
| 1307 | bool HexagonFrameLowering::updateExitPaths(MachineBasicBlock &MBB, |
| 1308 | MachineBasicBlock &RestoreB, BitVector &DoneT, BitVector &DoneF, |
| 1309 | BitVector &Path) const { |
| 1310 | assert(MBB.getNumber() >= 0); |
| 1311 | unsigned BN = MBB.getNumber(); |
| 1312 | if (Path[BN] || DoneF[BN]) |
| 1313 | return false; |
| 1314 | if (DoneT[BN]) |
| 1315 | return true; |
| 1316 | |
| 1317 | auto &CSI = MBB.getParent()->getFrameInfo().getCalleeSavedInfo(); |
| 1318 | |
| 1319 | Path[BN] = true; |
| 1320 | bool ReachedExit = false; |
| 1321 | for (auto &SB : MBB.successors()) |
| 1322 | ReachedExit |= updateExitPaths(MBB&: *SB, RestoreB, DoneT, DoneF, Path); |
| 1323 | |
| 1324 | if (!MBB.empty() && MBB.back().isReturn()) { |
| 1325 | // Add implicit uses of all callee-saved registers to the reached |
| 1326 | // return instructions. This is to prevent the anti-dependency breaker |
| 1327 | // from renaming these registers. |
| 1328 | MachineInstr &RetI = MBB.back(); |
| 1329 | if (!isRestoreCall(Opc: RetI.getOpcode())) |
| 1330 | for (auto &R : CSI) |
| 1331 | RetI.addOperand(Op: MachineOperand::CreateReg(Reg: R.getReg(), isDef: false, isImp: true)); |
| 1332 | ReachedExit = true; |
| 1333 | } |
| 1334 | |
| 1335 | // We don't want to add unnecessary live-ins to the restore block: since |
| 1336 | // the callee-saved registers are being defined in it, the entry of the |
| 1337 | // restore block cannot be on the path from the definitions to any exit. |
| 1338 | if (ReachedExit && &MBB != &RestoreB) { |
| 1339 | for (auto &R : CSI) |
| 1340 | if (!MBB.isLiveIn(Reg: R.getReg())) |
| 1341 | MBB.addLiveIn(PhysReg: R.getReg()); |
| 1342 | DoneT[BN] = true; |
| 1343 | } |
| 1344 | if (!ReachedExit) |
| 1345 | DoneF[BN] = true; |
| 1346 | |
| 1347 | Path[BN] = false; |
| 1348 | return ReachedExit; |
| 1349 | } |
| 1350 | |
| 1351 | static std::optional<MachineBasicBlock::iterator> |
| 1352 | findCFILocation(MachineBasicBlock &B) { |
| 1353 | // The CFI instructions need to be inserted right after allocframe. |
| 1354 | // An exception to this is a situation where allocframe is bundled |
| 1355 | // with a call: then the CFI instructions need to be inserted before |
| 1356 | // the packet with the allocframe+call (in case the call throws an |
| 1357 | // exception). |
| 1358 | auto End = B.instr_end(); |
| 1359 | |
| 1360 | for (MachineInstr &I : B) { |
| 1361 | MachineBasicBlock::iterator It = I.getIterator(); |
| 1362 | if (!I.isBundle()) { |
| 1363 | if (I.getOpcode() == Hexagon::S2_allocframe) |
| 1364 | return std::next(x: It); |
| 1365 | continue; |
| 1366 | } |
| 1367 | // I is a bundle. |
| 1368 | bool HasCall = false, HasAllocFrame = false; |
| 1369 | auto T = It.getInstrIterator(); |
| 1370 | while (++T != End && T->isBundled()) { |
| 1371 | if (T->getOpcode() == Hexagon::S2_allocframe) |
| 1372 | HasAllocFrame = true; |
| 1373 | else if (T->isCall()) |
| 1374 | HasCall = true; |
| 1375 | } |
| 1376 | if (HasAllocFrame) |
| 1377 | return HasCall ? It : std::next(x: It); |
| 1378 | } |
| 1379 | return std::nullopt; |
| 1380 | } |
| 1381 | |
| 1382 | void HexagonFrameLowering::insertCFIInstructions(MachineFunction &MF) const { |
| 1383 | for (auto &B : MF) |
| 1384 | if (auto At = findCFILocation(B)) |
| 1385 | insertCFIInstructionsAt(MBB&: B, At: *At); |
| 1386 | } |
| 1387 | |
| 1388 | void HexagonFrameLowering::insertCFIInstructionsAt(MachineBasicBlock &MBB, |
| 1389 | MachineBasicBlock::iterator At) const { |
| 1390 | MachineFunction &MF = *MBB.getParent(); |
| 1391 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 1392 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1393 | auto &HII = *HST.getInstrInfo(); |
| 1394 | auto &HRI = *HST.getRegisterInfo(); |
| 1395 | |
| 1396 | // If CFI instructions have debug information attached, something goes |
| 1397 | // wrong with the final assembly generation: the prolog_end is placed |
| 1398 | // in a wrong location. |
| 1399 | DebugLoc DL; |
| 1400 | const MCInstrDesc &CFID = HII.get(Opcode: TargetOpcode::CFI_INSTRUCTION); |
| 1401 | |
| 1402 | MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); |
| 1403 | bool HasFP = hasFP(MF); |
| 1404 | |
| 1405 | if (HasFP) { |
| 1406 | unsigned DwFPReg = HRI.getDwarfRegNum(Reg: HRI.getFrameRegister(), isEH: true); |
| 1407 | unsigned DwRAReg = HRI.getDwarfRegNum(Reg: HRI.getRARegister(), isEH: true); |
| 1408 | |
| 1409 | // Define CFA via an offset from the value of FP. |
| 1410 | // |
| 1411 | // -8 -4 0 (SP) |
| 1412 | // --+----+----+--------------------- |
| 1413 | // | FP | LR | increasing addresses --> |
| 1414 | // --+----+----+--------------------- |
| 1415 | // | +-- Old SP (before allocframe) |
| 1416 | // +-- New FP (after allocframe) |
| 1417 | // |
| 1418 | // MCCFIInstruction::cfiDefCfa adds the offset from the register. |
| 1419 | // MCCFIInstruction::createOffset takes the offset without sign change. |
| 1420 | auto DefCfa = MCCFIInstruction::cfiDefCfa(L: FrameLabel, Register: DwFPReg, Offset: 8); |
| 1421 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1422 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: DefCfa)); |
| 1423 | // R31 (return addr) = CFA - 4 |
| 1424 | auto OffR31 = MCCFIInstruction::createOffset(L: FrameLabel, Register: DwRAReg, Offset: -4); |
| 1425 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1426 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffR31)); |
| 1427 | // R30 (frame ptr) = CFA - 8 |
| 1428 | auto OffR30 = MCCFIInstruction::createOffset(L: FrameLabel, Register: DwFPReg, Offset: -8); |
| 1429 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1430 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffR30)); |
| 1431 | } |
| 1432 | |
| 1433 | static const MCPhysReg RegsToMove[] = { |
| 1434 | Hexagon::R1, Hexagon::R0, Hexagon::R3, Hexagon::R2, |
| 1435 | Hexagon::R17, Hexagon::R16, Hexagon::R19, Hexagon::R18, |
| 1436 | Hexagon::R21, Hexagon::R20, Hexagon::R23, Hexagon::R22, |
| 1437 | Hexagon::R25, Hexagon::R24, Hexagon::R27, Hexagon::R26, |
| 1438 | Hexagon::D0, Hexagon::D1, Hexagon::D8, Hexagon::D9, |
| 1439 | Hexagon::D10, Hexagon::D11, Hexagon::D12, Hexagon::D13 |
| 1440 | }; |
| 1441 | |
| 1442 | const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
| 1443 | |
| 1444 | for (MCPhysReg Reg : RegsToMove) { |
| 1445 | auto IfR = [Reg] (const CalleeSavedInfo &C) -> bool { |
| 1446 | return C.getReg() == Reg; |
| 1447 | }; |
| 1448 | auto F = find_if(Range: CSI, P: IfR); |
| 1449 | if (F == CSI.end()) |
| 1450 | continue; |
| 1451 | |
| 1452 | int64_t Offset; |
| 1453 | if (HasFP) { |
| 1454 | // If the function has a frame pointer (i.e. has an allocframe), |
| 1455 | // then the CFA has been defined in terms of FP. Any offsets in |
| 1456 | // the following CFI instructions have to be defined relative |
| 1457 | // to FP, which points to the bottom of the stack frame. |
| 1458 | // The function getFrameIndexReference can still choose to use SP |
| 1459 | // for the offset calculation, so we cannot simply call it here. |
| 1460 | // Instead, get the offset (relative to the FP) directly. |
| 1461 | Offset = MFI.getObjectOffset(ObjectIdx: F->getFrameIdx()); |
| 1462 | } else { |
| 1463 | Register FrameReg; |
| 1464 | Offset = |
| 1465 | getFrameIndexReference(MF, FI: F->getFrameIdx(), FrameReg).getFixed(); |
| 1466 | } |
| 1467 | // Subtract 8 to make room for R30 and R31, which are added above. |
| 1468 | Offset -= 8; |
| 1469 | |
| 1470 | if (Reg < Hexagon::D0 || Reg > Hexagon::D15) { |
| 1471 | unsigned DwarfReg = HRI.getDwarfRegNum(Reg, isEH: true); |
| 1472 | auto OffReg = MCCFIInstruction::createOffset(L: FrameLabel, Register: DwarfReg, |
| 1473 | Offset); |
| 1474 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1475 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffReg)); |
| 1476 | } else { |
| 1477 | // Split the double regs into subregs, and generate appropriate |
| 1478 | // cfi_offsets. |
| 1479 | // The only reason, we are split double regs is, llvm-mc does not |
| 1480 | // understand paired registers for cfi_offset. |
| 1481 | // Eg .cfi_offset r1:0, -64 |
| 1482 | |
| 1483 | Register HiReg = HRI.getSubReg(Reg, Idx: Hexagon::isub_hi); |
| 1484 | Register LoReg = HRI.getSubReg(Reg, Idx: Hexagon::isub_lo); |
| 1485 | unsigned HiDwarfReg = HRI.getDwarfRegNum(Reg: HiReg, isEH: true); |
| 1486 | unsigned LoDwarfReg = HRI.getDwarfRegNum(Reg: LoReg, isEH: true); |
| 1487 | auto OffHi = MCCFIInstruction::createOffset(L: FrameLabel, Register: HiDwarfReg, |
| 1488 | Offset: Offset+4); |
| 1489 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1490 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffHi)); |
| 1491 | auto OffLo = MCCFIInstruction::createOffset(L: FrameLabel, Register: LoDwarfReg, |
| 1492 | Offset); |
| 1493 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID) |
| 1494 | .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffLo)); |
| 1495 | } |
| 1496 | } |
| 1497 | |
| 1498 | // HLX callee-saved registers are spilled to the VTCM stack, which is |
| 1499 | // addressed by VSP, not by the CFA. Describe them as |
| 1500 | // DW_CFA_expression <reg>, DW_OP_bregx <vsp>, <offset> |
| 1501 | // where the offset is relative to the VSP after vallocframe, which stays |
| 1502 | // the same in the function body. Preserved registers that are not spilled |
| 1503 | // keep their value. |
| 1504 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 1505 | unsigned DwVSPReg = HRI.getDwarfRegNum(Reg: Hexagon::VSP, isEH: true); |
| 1506 | int64_t VTCMFrameSize = HMFI.getVTCMFrameInfo().getStackSize(); |
| 1507 | for (const MCPhysReg *R = HRI.getCalleeSavedRegs(MF: &MF); *R; ++R) { |
| 1508 | if (!Hexagon::HlxLRRegClass.contains(Reg: *R)) |
| 1509 | continue; |
| 1510 | unsigned DwarfReg = HRI.getDwarfRegNum(Reg: *R, isEH: true); |
| 1511 | auto F = find_if( |
| 1512 | Range: CSI, P: [R](const CalleeSavedInfo &C) { return C.getReg() == *R; }); |
| 1513 | if (F == CSI.end()) { |
| 1514 | auto Same = MCCFIInstruction::createSameValue(L: FrameLabel, Register: DwarfReg); |
| 1515 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID).addCFIIndex(CFIIndex: MF.addFrameInst(Inst: Same)); |
| 1516 | continue; |
| 1517 | } |
| 1518 | assert(HMFI.isVTCMStackIndex(F->getFrameIdx()) && |
| 1519 | "HLX register not spilled to the VTCM stack" ); |
| 1520 | int64_t Offset = |
| 1521 | VTCMFrameSize + HMFI.getMappedVTCMObjectOffset(ObjectIdx: F->getFrameIdx()); |
| 1522 | uint8_t Buf[16]; |
| 1523 | SmallString<16> Expr, Vals; |
| 1524 | Expr.push_back(Elt: dwarf::DW_OP_bregx); |
| 1525 | Expr.append(in_start: Buf, in_end: Buf + encodeULEB128(Value: DwVSPReg, p: Buf)); |
| 1526 | Expr.append(in_start: Buf, in_end: Buf + encodeSLEB128(Value: Offset, p: Buf)); |
| 1527 | Vals.push_back(Elt: dwarf::DW_CFA_expression); |
| 1528 | Vals.append(in_start: Buf, in_end: Buf + encodeULEB128(Value: DwarfReg, p: Buf)); |
| 1529 | Vals.append(in_start: Buf, in_end: Buf + encodeULEB128(Value: Expr.size(), p: Buf)); |
| 1530 | Vals.append(RHS: Expr); |
| 1531 | std::string = |
| 1532 | (Twine(HRI.getRegAsmName(Reg: *R)) + " at vsp + " + Twine(Offset)).str(); |
| 1533 | auto Esc = |
| 1534 | MCCFIInstruction::createEscape(L: FrameLabel, Vals, Loc: SMLoc(), Comment); |
| 1535 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID).addCFIIndex(CFIIndex: MF.addFrameInst(Inst: Esc)); |
| 1536 | } |
| 1537 | |
| 1538 | // The caller's VSP, saved by insertPrologueInBlock. |
| 1539 | if (int VSPFI = HMFI.getVSPFrameIndex(); VSPFI >= 0) { |
| 1540 | assert(HasFP && "A VTCM frame requires a frame pointer" ); |
| 1541 | auto OffVSP = MCCFIInstruction::createOffset( |
| 1542 | L: FrameLabel, Register: DwVSPReg, Offset: MFI.getObjectOffset(ObjectIdx: VSPFI) - 8); |
| 1543 | BuildMI(BB&: MBB, I: At, MIMD: DL, MCID: CFID).addCFIIndex(CFIIndex: MF.addFrameInst(Inst: OffVSP)); |
| 1544 | } |
| 1545 | } |
| 1546 | |
| 1547 | bool HexagonFrameLowering::hasFPImpl(const MachineFunction &MF) const { |
| 1548 | auto &MFI = MF.getFrameInfo(); |
| 1549 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 1550 | bool = HRI.hasStackRealignment(MF); |
| 1551 | bool HasAlloca = MFI.hasVarSizedObjects(); |
| 1552 | |
| 1553 | // Insert ALLOCFRAME if we need to or at -O0 for the debugger. Think |
| 1554 | // that this shouldn't be required, but doing so now because gcc does and |
| 1555 | // gdb can't break at the start of the function without it. Will remove if |
| 1556 | // this turns out to be a gdb bug. |
| 1557 | // |
| 1558 | if (MF.getTarget().getOptLevel() == CodeGenOptLevel::None) |
| 1559 | return true; |
| 1560 | |
| 1561 | // By default we want to use SP (since it's always there). FP requires |
| 1562 | // some setup (i.e. ALLOCFRAME). |
| 1563 | // Both, alloca and stack alignment modify the stack pointer by an |
| 1564 | // undetermined value, so we need to save it at the entry to the function |
| 1565 | // (i.e. use allocframe). |
| 1566 | if (HasAlloca || HasExtraAlign) |
| 1567 | return true; |
| 1568 | |
| 1569 | // If FP-elimination is disabled, we have to use FP. This must not be |
| 1570 | // gated on stack size: the user/ABI-requested frame pointer is needed |
| 1571 | // regardless of whether the function currently has a stack frame. |
| 1572 | // Every other target checks DisableFramePointerElim unconditionally. |
| 1573 | if (MF.disableFramePointerElim() || !EliminateFramePointer) |
| 1574 | return true; |
| 1575 | |
| 1576 | if (MFI.getStackSize() > 0) { |
| 1577 | if (EnableStackOVFSanitizer) |
| 1578 | return true; |
| 1579 | } |
| 1580 | |
| 1581 | const auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 1582 | if (HMFI.getVTCMFrameInfo().getStackSize()) |
| 1583 | return true; |
| 1584 | if ((MFI.hasCalls() && !enableAllocFrameElim(MF)) || HMFI.hasClobberLR()) |
| 1585 | return true; |
| 1586 | |
| 1587 | return false; |
| 1588 | } |
| 1589 | |
| 1590 | enum SpillKind { |
| 1591 | SK_ToMem, |
| 1592 | SK_FromMem, |
| 1593 | SK_FromMemTailcall |
| 1594 | }; |
| 1595 | |
| 1596 | static const char *getSpillFunctionFor(Register MaxReg, SpillKind SpillType, |
| 1597 | bool Stkchk = false) { |
| 1598 | const char * V4SpillToMemoryFunctions[] = { |
| 1599 | "__save_r16_through_r17" , |
| 1600 | "__save_r16_through_r19" , |
| 1601 | "__save_r16_through_r21" , |
| 1602 | "__save_r16_through_r23" , |
| 1603 | "__save_r16_through_r25" , |
| 1604 | "__save_r16_through_r27" }; |
| 1605 | |
| 1606 | const char * V4SpillToMemoryStkchkFunctions[] = { |
| 1607 | "__save_r16_through_r17_stkchk" , |
| 1608 | "__save_r16_through_r19_stkchk" , |
| 1609 | "__save_r16_through_r21_stkchk" , |
| 1610 | "__save_r16_through_r23_stkchk" , |
| 1611 | "__save_r16_through_r25_stkchk" , |
| 1612 | "__save_r16_through_r27_stkchk" }; |
| 1613 | |
| 1614 | const char * V4SpillFromMemoryFunctions[] = { |
| 1615 | "__restore_r16_through_r17_and_deallocframe" , |
| 1616 | "__restore_r16_through_r19_and_deallocframe" , |
| 1617 | "__restore_r16_through_r21_and_deallocframe" , |
| 1618 | "__restore_r16_through_r23_and_deallocframe" , |
| 1619 | "__restore_r16_through_r25_and_deallocframe" , |
| 1620 | "__restore_r16_through_r27_and_deallocframe" }; |
| 1621 | |
| 1622 | const char * V4SpillFromMemoryTailcallFunctions[] = { |
| 1623 | "__restore_r16_through_r17_and_deallocframe_before_tailcall" , |
| 1624 | "__restore_r16_through_r19_and_deallocframe_before_tailcall" , |
| 1625 | "__restore_r16_through_r21_and_deallocframe_before_tailcall" , |
| 1626 | "__restore_r16_through_r23_and_deallocframe_before_tailcall" , |
| 1627 | "__restore_r16_through_r25_and_deallocframe_before_tailcall" , |
| 1628 | "__restore_r16_through_r27_and_deallocframe_before_tailcall" |
| 1629 | }; |
| 1630 | |
| 1631 | const char **SpillFunc = nullptr; |
| 1632 | |
| 1633 | switch(SpillType) { |
| 1634 | case SK_ToMem: |
| 1635 | SpillFunc = Stkchk ? V4SpillToMemoryStkchkFunctions |
| 1636 | : V4SpillToMemoryFunctions; |
| 1637 | break; |
| 1638 | case SK_FromMem: |
| 1639 | SpillFunc = V4SpillFromMemoryFunctions; |
| 1640 | break; |
| 1641 | case SK_FromMemTailcall: |
| 1642 | SpillFunc = V4SpillFromMemoryTailcallFunctions; |
| 1643 | break; |
| 1644 | } |
| 1645 | assert(SpillFunc && "Unknown spill kind" ); |
| 1646 | |
| 1647 | // Spill all callee-saved registers up to the highest register used. |
| 1648 | switch (MaxReg) { |
| 1649 | case Hexagon::R17: |
| 1650 | return SpillFunc[0]; |
| 1651 | case Hexagon::R19: |
| 1652 | return SpillFunc[1]; |
| 1653 | case Hexagon::R21: |
| 1654 | return SpillFunc[2]; |
| 1655 | case Hexagon::R23: |
| 1656 | return SpillFunc[3]; |
| 1657 | case Hexagon::R25: |
| 1658 | return SpillFunc[4]; |
| 1659 | case Hexagon::R27: |
| 1660 | return SpillFunc[5]; |
| 1661 | default: |
| 1662 | llvm_unreachable("Unhandled maximum callee save register" ); |
| 1663 | } |
| 1664 | return nullptr; |
| 1665 | } |
| 1666 | |
| 1667 | StackOffset |
| 1668 | HexagonFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, |
| 1669 | Register &FrameReg) const { |
| 1670 | auto &MFI = MF.getFrameInfo(); |
| 1671 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 1672 | |
| 1673 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 1674 | if (HMFI.isVTCMStackIndex(ObjectIdx: FI)) { |
| 1675 | const MachineFrameInfo &VTCMFrameInfo = HMFI.getVTCMFrameInfo(); |
| 1676 | FrameReg = HRI.getVTCMStackRegister(); |
| 1677 | return StackOffset::getFixed(Fixed: VTCMFrameInfo.getStackSize() + |
| 1678 | HMFI.getMappedVTCMObjectOffset(ObjectIdx: FI)); |
| 1679 | } |
| 1680 | |
| 1681 | int Offset = MFI.getObjectOffset(ObjectIdx: FI); |
| 1682 | bool HasAlloca = MFI.hasVarSizedObjects(); |
| 1683 | bool = HRI.hasStackRealignment(MF); |
| 1684 | bool NoOpt = MF.getTarget().getOptLevel() == CodeGenOptLevel::None; |
| 1685 | |
| 1686 | unsigned FrameSize = MFI.getStackSize(); |
| 1687 | Register SP = HRI.getStackRegister(); |
| 1688 | Register FP = HRI.getFrameRegister(); |
| 1689 | Register AP = HMFI.getStackAlignBaseReg(); |
| 1690 | // It may happen that AP will be absent even HasAlloca && HasExtraAlign |
| 1691 | // is true. HasExtraAlign may be set because of vector spills, without |
| 1692 | // aligned locals or aligned outgoing function arguments. Since vector |
| 1693 | // spills will ultimately be "unaligned", it is safe to use FP as the |
| 1694 | // base register. |
| 1695 | // In fact, in such a scenario the stack is actually not required to be |
| 1696 | // aligned, although it may end up being aligned anyway, since this |
| 1697 | // particular case is not easily detectable. The alignment will be |
| 1698 | // unnecessary, but not incorrect. |
| 1699 | // Unfortunately there is no quick way to verify that the above is |
| 1700 | // indeed the case (and that it's not a result of an error), so just |
| 1701 | // assume that missing AP will be replaced by FP. |
| 1702 | // (A better fix would be to rematerialize AP from FP and always align |
| 1703 | // vector spills.) |
| 1704 | bool UseFP = false, UseAP = false; // Default: use SP (except at -O0). |
| 1705 | // Use FP at -O0, except when there are objects with extra alignment. |
| 1706 | // That additional alignment requirement may cause a pad to be inserted, |
| 1707 | // which will make it impossible to use FP to access objects located |
| 1708 | // past the pad. |
| 1709 | if (NoOpt && !HasExtraAlign) |
| 1710 | UseFP = true; |
| 1711 | if (MFI.isFixedObjectIndex(ObjectIdx: FI) || MFI.isObjectPreAllocated(ObjectIdx: FI)) { |
| 1712 | // Fixed and preallocated objects will be located before any padding |
| 1713 | // so FP must be used to access them. |
| 1714 | UseFP |= (HasAlloca || HasExtraAlign); |
| 1715 | } else { |
| 1716 | if (HasAlloca) { |
| 1717 | if (HasExtraAlign) |
| 1718 | UseAP = true; |
| 1719 | else |
| 1720 | UseFP = true; |
| 1721 | } |
| 1722 | } |
| 1723 | |
| 1724 | // If FP was picked, then there had better be FP. |
| 1725 | bool HasFP = hasFP(MF); |
| 1726 | assert((HasFP || !UseFP) && "This function must have frame pointer" ); |
| 1727 | |
| 1728 | // Having FP implies allocframe. Allocframe will store extra 8 bytes: |
| 1729 | // FP/LR. If the base register is used to access an object across these |
| 1730 | // 8 bytes, then the offset will need to be adjusted by 8. |
| 1731 | // |
| 1732 | // After allocframe: |
| 1733 | // HexagonISelLowering adds 8 to ---+ |
| 1734 | // the offsets of all stack-based | |
| 1735 | // arguments (*) | |
| 1736 | // | |
| 1737 | // getObjectOffset < 0 0 8 getObjectOffset >= 8 |
| 1738 | // ------------------------+-----+------------------------> increasing |
| 1739 | // <local objects> |FP/LR| <input arguments> addresses |
| 1740 | // -----------------+------+-----+------------------------> |
| 1741 | // | | |
| 1742 | // SP/AP point --+ +-- FP points here (**) |
| 1743 | // somewhere on |
| 1744 | // this side of FP/LR |
| 1745 | // |
| 1746 | // (*) See LowerFormalArguments. The FP/LR is assumed to be present. |
| 1747 | // (**) *FP == old-FP. FP+0..7 are the bytes of FP/LR. |
| 1748 | |
| 1749 | // The lowering assumes that FP/LR is present, and so the offsets of |
| 1750 | // the formal arguments start at 8. If FP/LR is not there we need to |
| 1751 | // reduce the offset by 8. |
| 1752 | if (Offset > 0 && !HasFP) |
| 1753 | Offset -= 8; |
| 1754 | |
| 1755 | if (UseFP) |
| 1756 | FrameReg = FP; |
| 1757 | else if (UseAP) |
| 1758 | FrameReg = AP; |
| 1759 | else |
| 1760 | FrameReg = SP; |
| 1761 | |
| 1762 | // Calculate the actual offset in the instruction. If there is no FP |
| 1763 | // (in other words, no allocframe), then SP will not be adjusted (i.e. |
| 1764 | // there will be no SP -= FrameSize), so the frame size should not be |
| 1765 | // added to the calculated offset. |
| 1766 | int RealOffset = Offset; |
| 1767 | if (!UseFP && !UseAP) |
| 1768 | RealOffset = FrameSize+Offset; |
| 1769 | return StackOffset::getFixed(Fixed: RealOffset); |
| 1770 | } |
| 1771 | |
| 1772 | MachineBasicBlock::iterator HexagonFrameLowering::insertCSRSpillsInBlock( |
| 1773 | MachineBasicBlock &MBB, const CSIVect &CSI, const HexagonRegisterInfo &HRI, |
| 1774 | bool &PrologueStubs) const { |
| 1775 | MachineBasicBlock::iterator MI = MBB.begin(); |
| 1776 | if (CSI.empty()) |
| 1777 | return MI; |
| 1778 | |
| 1779 | PrologueStubs = false; |
| 1780 | MachineFunction &MF = *MBB.getParent(); |
| 1781 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1782 | auto &HII = *HST.getInstrInfo(); |
| 1783 | |
| 1784 | if (useSpillFunction(MF, CSI)) { |
| 1785 | PrologueStubs = true; |
| 1786 | Register MaxReg = getMaxCalleeSavedReg(CSI, TRI: HRI); |
| 1787 | bool StkOvrFlowEnabled = EnableStackOVFSanitizer; |
| 1788 | const char *SpillFun = getSpillFunctionFor(MaxReg, SpillType: SK_ToMem, |
| 1789 | Stkchk: StkOvrFlowEnabled); |
| 1790 | auto &HTM = static_cast<const HexagonTargetMachine&>(MF.getTarget()); |
| 1791 | bool IsPIC = HTM.isPositionIndependent(); |
| 1792 | bool LongCalls = HST.useLongCalls() || EnableSaveRestoreLong; |
| 1793 | |
| 1794 | // Call spill function. |
| 1795 | DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc(); |
| 1796 | unsigned SpillOpc; |
| 1797 | if (StkOvrFlowEnabled) { |
| 1798 | if (LongCalls) |
| 1799 | SpillOpc = IsPIC ? Hexagon::SAVE_REGISTERS_CALL_V4STK_EXT_PIC |
| 1800 | : Hexagon::SAVE_REGISTERS_CALL_V4STK_EXT; |
| 1801 | else |
| 1802 | SpillOpc = IsPIC ? Hexagon::SAVE_REGISTERS_CALL_V4STK_PIC |
| 1803 | : Hexagon::SAVE_REGISTERS_CALL_V4STK; |
| 1804 | } else { |
| 1805 | if (LongCalls) |
| 1806 | SpillOpc = IsPIC ? Hexagon::SAVE_REGISTERS_CALL_V4_EXT_PIC |
| 1807 | : Hexagon::SAVE_REGISTERS_CALL_V4_EXT; |
| 1808 | else |
| 1809 | SpillOpc = IsPIC ? Hexagon::SAVE_REGISTERS_CALL_V4_PIC |
| 1810 | : Hexagon::SAVE_REGISTERS_CALL_V4; |
| 1811 | } |
| 1812 | |
| 1813 | MachineInstr *SaveRegsCall = |
| 1814 | BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: SpillOpc)) |
| 1815 | .addExternalSymbol(FnName: SpillFun); |
| 1816 | |
| 1817 | // Add callee-saved registers as use. |
| 1818 | addCalleeSaveRegistersAsImpOperand(MI: SaveRegsCall, CSI, IsDef: false, IsKill: true); |
| 1819 | // Add live in registers. |
| 1820 | for (const CalleeSavedInfo &I : CSI) |
| 1821 | MBB.addLiveIn(PhysReg: I.getReg()); |
| 1822 | } else { |
| 1823 | for (const CalleeSavedInfo &I : CSI) { |
| 1824 | MCRegister Reg = I.getReg(); |
| 1825 | // Add live in registers. We treat eh_return callee saved register r0 - r3 |
| 1826 | // specially. They are not really callee saved registers as they are not |
| 1827 | // supposed to be killed. |
| 1828 | bool IsKill = !HRI.isEHReturnCalleeSaveReg(Reg); |
| 1829 | int FI = I.getFrameIdx(); |
| 1830 | const TargetRegisterClass *RC = HRI.getMinimalPhysRegClass(Reg); |
| 1831 | HII.storeRegToStackSlot(MBB, MBBI: MI, SrcReg: Reg, isKill: IsKill, FrameIndex: FI, RC, VReg: Register()); |
| 1832 | if (IsKill) |
| 1833 | MBB.addLiveIn(PhysReg: Reg); |
| 1834 | } |
| 1835 | } |
| 1836 | |
| 1837 | return MI; |
| 1838 | } |
| 1839 | |
| 1840 | bool HexagonFrameLowering::insertCSRRestoresInBlock(MachineBasicBlock &MBB, |
| 1841 | const CSIVect &CSI, const HexagonRegisterInfo &HRI) const { |
| 1842 | if (CSI.empty()) |
| 1843 | return false; |
| 1844 | |
| 1845 | MachineBasicBlock::iterator MI = MBB.getFirstTerminator(); |
| 1846 | MachineFunction &MF = *MBB.getParent(); |
| 1847 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1848 | auto &HII = *HST.getInstrInfo(); |
| 1849 | |
| 1850 | if (useRestoreFunction(MF, CSI)) { |
| 1851 | bool HasTC = hasTailCall(MBB) || !hasReturn(MBB); |
| 1852 | Register MaxR = getMaxCalleeSavedReg(CSI, TRI: HRI); |
| 1853 | SpillKind Kind = HasTC ? SK_FromMemTailcall : SK_FromMem; |
| 1854 | const char *RestoreFn = getSpillFunctionFor(MaxReg: MaxR, SpillType: Kind); |
| 1855 | auto &HTM = static_cast<const HexagonTargetMachine&>(MF.getTarget()); |
| 1856 | bool IsPIC = HTM.isPositionIndependent(); |
| 1857 | bool LongCalls = HST.useLongCalls() || EnableSaveRestoreLong; |
| 1858 | |
| 1859 | // Call spill function. |
| 1860 | DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() |
| 1861 | : MBB.findDebugLoc(MBBI: MBB.end()); |
| 1862 | MachineInstr *DeallocCall = nullptr; |
| 1863 | |
| 1864 | if (HasTC) { |
| 1865 | unsigned RetOpc; |
| 1866 | if (LongCalls) |
| 1867 | RetOpc = IsPIC ? Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT_PIC |
| 1868 | : Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_EXT; |
| 1869 | else |
| 1870 | RetOpc = IsPIC ? Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4_PIC |
| 1871 | : Hexagon::RESTORE_DEALLOC_BEFORE_TAILCALL_V4; |
| 1872 | DeallocCall = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: HII.get(Opcode: RetOpc)) |
| 1873 | .addExternalSymbol(FnName: RestoreFn); |
| 1874 | } else { |
| 1875 | // The block has a return. |
| 1876 | MachineBasicBlock::iterator It = MBB.getFirstTerminator(); |
| 1877 | assert(It->isReturn() && std::next(It) == MBB.end()); |
| 1878 | unsigned RetOpc; |
| 1879 | if (LongCalls) |
| 1880 | RetOpc = IsPIC ? Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT_PIC |
| 1881 | : Hexagon::RESTORE_DEALLOC_RET_JMP_V4_EXT; |
| 1882 | else |
| 1883 | RetOpc = IsPIC ? Hexagon::RESTORE_DEALLOC_RET_JMP_V4_PIC |
| 1884 | : Hexagon::RESTORE_DEALLOC_RET_JMP_V4; |
| 1885 | MachineInstrBuilder MIB = |
| 1886 | BuildMI(BB&: MBB, I: It, MIMD: DL, MCID: HII.get(Opcode: RetOpc)).addExternalSymbol(FnName: RestoreFn); |
| 1887 | // Transfer the function live-out registers, avoiding duplicating the pc |
| 1888 | // implicit-def. |
| 1889 | for (const MachineOperand &MO : drop_begin(RangeOrContainer: It->implicit_operands())) |
| 1890 | MIB.add(MO); |
| 1891 | DeallocCall = MIB; |
| 1892 | } |
| 1893 | addCalleeSaveRegistersAsImpOperand(MI: DeallocCall, CSI, IsDef: true, IsKill: false); |
| 1894 | return true; |
| 1895 | } |
| 1896 | |
| 1897 | for (const CalleeSavedInfo &I : CSI) { |
| 1898 | MCRegister Reg = I.getReg(); |
| 1899 | const TargetRegisterClass *RC = HRI.getMinimalPhysRegClass(Reg); |
| 1900 | int FI = I.getFrameIdx(); |
| 1901 | HII.loadRegFromStackSlot(MBB, MBBI: MI, DestReg: Reg, FrameIndex: FI, RC, VReg: Register()); |
| 1902 | } |
| 1903 | |
| 1904 | return true; |
| 1905 | } |
| 1906 | |
| 1907 | MachineBasicBlock::iterator HexagonFrameLowering::eliminateCallFramePseudoInstr( |
| 1908 | MachineFunction &MF, MachineBasicBlock &MBB, |
| 1909 | MachineBasicBlock::iterator I) const { |
| 1910 | MachineInstr &MI = *I; |
| 1911 | unsigned Opc = MI.getOpcode(); |
| 1912 | (void)Opc; // Silence compiler warning. |
| 1913 | assert((Opc == Hexagon::ADJCALLSTACKDOWN || Opc == Hexagon::ADJCALLSTACKUP) && |
| 1914 | "Cannot handle this call frame pseudo instruction" ); |
| 1915 | return MBB.erase(I); |
| 1916 | } |
| 1917 | |
| 1918 | void HexagonFrameLowering::computeVTCMStackOffsets(MachineFunction &MF) const { |
| 1919 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 1920 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1921 | const auto &HRI = *HST.getRegisterInfo(); |
| 1922 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 1923 | MachineFrameInfo &VTCMFrameInfo = HMFI.getVTCMFrameInfo(); |
| 1924 | const int64_t HlxLRSize = HRI.getSpillSize(RC: Hexagon::HlxLRRegClass); |
| 1925 | const int64_t HlxLLRSize = HRI.getSpillSize(RC: Hexagon::HlxLLRRegClass); |
| 1926 | |
| 1927 | // Incoming HLX stack arguments are just above the VTCM frame. NoAlloc |
| 1928 | // fixed objects are incoming HLX stack arguments (see LowerFormalArguments). |
| 1929 | for (int I = MFI.getObjectIndexBegin(); I != 0; ++I) { |
| 1930 | if (MFI.getStackID(ObjectIdx: I) != TargetStackID::NoAlloc) |
| 1931 | continue; |
| 1932 | HMFI.moveToVTCMStack(From: I, To: VTCMFrameInfo.CreateFixedObject( |
| 1933 | Size: MFI.getObjectSize(ObjectIdx: I), SPOffset: MFI.getObjectOffset(ObjectIdx: I), |
| 1934 | /*IsImmutable=*/true)); |
| 1935 | } |
| 1936 | |
| 1937 | int64_t Offset = 0; |
| 1938 | for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) { |
| 1939 | bool IsHlxSpill = |
| 1940 | MFI.isSpillSlotObjectIndex(ObjectIdx: I) && (MFI.getObjectSize(ObjectIdx: I) == HlxLRSize || |
| 1941 | MFI.getObjectSize(ObjectIdx: I) == HlxLLRSize); |
| 1942 | if (MFI.isDeadObjectIndex(ObjectIdx: I) || MFI.isFixedObjectIndex(ObjectIdx: I) || |
| 1943 | (!IsHlxSpill && MFI.isSpillSlotObjectIndex(ObjectIdx: I)) || |
| 1944 | MFI.isVariableSizedObjectIndex(ObjectIdx: I) || MFI.isObjectPreAllocated(ObjectIdx: I)) |
| 1945 | continue; |
| 1946 | const AllocaInst *AI = MFI.getObjectAllocation(ObjectIdx: I); |
| 1947 | if (!IsHlxSpill && (!AI || !containsHLXObject(HST, Ty: AI->getAllocatedType()))) |
| 1948 | continue; |
| 1949 | |
| 1950 | int64_t Size = MFI.getObjectSize(ObjectIdx: I); |
| 1951 | Align Alignment = MFI.getObjectAlign(ObjectIdx: I); |
| 1952 | int VTCMIndex = |
| 1953 | IsHlxSpill ? VTCMFrameInfo.CreateSpillStackObject(Size, Alignment) |
| 1954 | : VTCMFrameInfo.CreateStackObject( |
| 1955 | Size, Alignment, isSpillSlot: false, Alloca: MFI.getObjectAllocation(ObjectIdx: I)); |
| 1956 | Offset += Size; |
| 1957 | VTCMFrameInfo.setObjectOffset(ObjectIdx: VTCMIndex, SPOffset: -Offset); |
| 1958 | HMFI.moveToVTCMStack(From: I, To: VTCMIndex); |
| 1959 | MFI.RemoveStackObject(ObjectIdx: I); |
| 1960 | } |
| 1961 | // Outgoing HLX stack arguments are stored at VSP+0 and up, below the |
| 1962 | // objects above. |
| 1963 | uint64_t StackSize = Offset + HMFI.getMaxVTCMCallFrameSize(); |
| 1964 | VTCMFrameInfo.setStackSize(StackSize); |
| 1965 | |
| 1966 | // The "hlx-vector-stack-limit" function attribute overrides the option. |
| 1967 | const Function &F = MF.getFunction(); |
| 1968 | uint64_t Limit = F.getFnAttributeAsParsedInteger(Kind: "hlx-vector-stack-limit" , |
| 1969 | Default: HLXVectorStackLimit); |
| 1970 | if (StackSize > Limit) |
| 1971 | F.getContext().diagnose(DI: DiagnosticInfoResourceLimit( |
| 1972 | F, "HLX vector stack size" , StackSize, Limit, DS_Error)); |
| 1973 | } |
| 1974 | |
| 1975 | void HexagonFrameLowering::processFunctionBeforeFrameFinalized( |
| 1976 | MachineFunction &MF, RegScavenger *RS) const { |
| 1977 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1978 | if (!HST.hasV85OpsOnly() || !HST.useHLXOps()) |
| 1979 | return; |
| 1980 | computeVTCMStackOffsets(MF); |
| 1981 | auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>(); |
| 1982 | if (SaveParentVSP && HMFI.getVTCMFrameInfo().getStackSize()) |
| 1983 | HMFI.setVSPFrameIndex( |
| 1984 | MF.getFrameInfo().CreateSpillStackObject(Size: 4, Alignment: Align(4))); |
| 1985 | } |
| 1986 | |
| 1987 | Align HexagonFrameLowering::getMaxStackAlign(const MachineFunction &MF) const { |
| 1988 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 1989 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 1990 | if (!HST.hasV85OpsOnly() || !HST.useHLXOps()) |
| 1991 | return MFI.getMaxAlign(); |
| 1992 | |
| 1993 | Align MaxAlign(1); |
| 1994 | for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E; |
| 1995 | ++I) { |
| 1996 | if (MFI.isDeadObjectIndex(ObjectIdx: I)) |
| 1997 | continue; |
| 1998 | MaxAlign = std::max(a: MaxAlign, b: MFI.getObjectAlign(ObjectIdx: I)); |
| 1999 | } |
| 2000 | return MaxAlign; |
| 2001 | } |
| 2002 | |
| 2003 | /// Returns true if there are no caller-saved registers available in class RC. |
| 2004 | static bool needToReserveScavengingSpillSlots(MachineFunction &MF, |
| 2005 | const HexagonRegisterInfo &HRI, const TargetRegisterClass *RC) { |
| 2006 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2007 | |
| 2008 | auto IsUsed = [&HRI,&MRI] (Register Reg) -> bool { |
| 2009 | for (MCRegAliasIterator AI(Reg, &HRI, true); AI.isValid(); ++AI) |
| 2010 | if (MRI.isPhysRegUsed(PhysReg: *AI)) |
| 2011 | return true; |
| 2012 | return false; |
| 2013 | }; |
| 2014 | |
| 2015 | // Check for an unused caller-saved register. Callee-saved registers |
| 2016 | // have become pristine by now. |
| 2017 | for (const MCPhysReg *P = HRI.getCallerSavedRegs(MF: &MF, RC); *P; ++P) |
| 2018 | if (!IsUsed(*P)) |
| 2019 | return false; |
| 2020 | |
| 2021 | // All caller-saved registers are used. |
| 2022 | return true; |
| 2023 | } |
| 2024 | |
| 2025 | #ifndef NDEBUG |
| 2026 | static void dump_registers(BitVector &Regs, const TargetRegisterInfo &TRI) { |
| 2027 | dbgs() << '{'; |
| 2028 | for (int x = Regs.find_first(); x >= 0; x = Regs.find_next(x)) { |
| 2029 | Register R = x; |
| 2030 | dbgs() << ' ' << printReg(R, &TRI); |
| 2031 | } |
| 2032 | dbgs() << " }" ; |
| 2033 | } |
| 2034 | #endif |
| 2035 | |
| 2036 | bool HexagonFrameLowering::assignCalleeSavedSpillSlots(MachineFunction &MF, |
| 2037 | const TargetRegisterInfo *TRI, std::vector<CalleeSavedInfo> &CSI) const { |
| 2038 | LLVM_DEBUG(dbgs() << __func__ << " on " << MF.getName() << '\n'); |
| 2039 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 2040 | BitVector SRegs(Hexagon::NUM_TARGET_REGS); |
| 2041 | |
| 2042 | // Generate a set of unique, callee-saved registers (SRegs), where each |
| 2043 | // register in the set is maximal in terms of sub-/super-register relation, |
| 2044 | // i.e. for each R in SRegs, no proper super-register of R is also in SRegs. |
| 2045 | |
| 2046 | // (1) For each callee-saved register, add that register and all of its |
| 2047 | // sub-registers to SRegs. |
| 2048 | LLVM_DEBUG(dbgs() << "Initial CS registers: {" ); |
| 2049 | for (const CalleeSavedInfo &I : CSI) { |
| 2050 | Register R = I.getReg(); |
| 2051 | LLVM_DEBUG(dbgs() << ' ' << printReg(R, TRI)); |
| 2052 | for (MCPhysReg SR : TRI->subregs_inclusive(Reg: R)) |
| 2053 | SRegs[SR] = true; |
| 2054 | } |
| 2055 | LLVM_DEBUG(dbgs() << " }\n" ); |
| 2056 | LLVM_DEBUG(dbgs() << "SRegs.1: " ; dump_registers(SRegs, *TRI); |
| 2057 | dbgs() << "\n" ); |
| 2058 | |
| 2059 | // (2) For each reserved register, remove that register and all of its |
| 2060 | // sub- and super-registers from SRegs. |
| 2061 | BitVector Reserved = TRI->getReservedRegs(MF); |
| 2062 | // Unreserve the stack align register: it is reserved for this function |
| 2063 | // only, it still needs to be saved/restored. |
| 2064 | Register AP = |
| 2065 | MF.getInfo<HexagonMachineFunctionInfo>()->getStackAlignBaseReg(); |
| 2066 | assert((!needsAligna(MF) || AP.isValid()) && |
| 2067 | "AP must be assigned before register allocation" ); |
| 2068 | if (AP.isValid()) { |
| 2069 | Reserved[AP] = false; |
| 2070 | // Unreserve super-regs if no other subregisters are reserved. |
| 2071 | for (MCPhysReg SP : TRI->superregs(Reg: AP)) { |
| 2072 | bool HasResSub = false; |
| 2073 | for (MCPhysReg SB : TRI->subregs(Reg: SP)) { |
| 2074 | if (!Reserved[SB]) |
| 2075 | continue; |
| 2076 | HasResSub = true; |
| 2077 | break; |
| 2078 | } |
| 2079 | if (!HasResSub) |
| 2080 | Reserved[SP] = false; |
| 2081 | } |
| 2082 | } |
| 2083 | |
| 2084 | for (int x = Reserved.find_first(); x >= 0; x = Reserved.find_next(Prev: x)) { |
| 2085 | Register R = x; |
| 2086 | for (MCPhysReg SR : TRI->superregs_inclusive(Reg: R)) |
| 2087 | SRegs[SR] = false; |
| 2088 | } |
| 2089 | LLVM_DEBUG(dbgs() << "Res: " ; dump_registers(Reserved, *TRI); |
| 2090 | dbgs() << "\n" ); |
| 2091 | LLVM_DEBUG(dbgs() << "SRegs.2: " ; dump_registers(SRegs, *TRI); |
| 2092 | dbgs() << "\n" ); |
| 2093 | |
| 2094 | // (3) Collect all registers that have at least one sub-register in SRegs, |
| 2095 | // and also have no sub-registers that are reserved. These will be the can- |
| 2096 | // didates for saving as a whole instead of their individual sub-registers. |
| 2097 | // (Saving R17:16 instead of R16 is fine, but only if R17 was not reserved.) |
| 2098 | // HLX registers are saved individually: there is no HLX pair spill. |
| 2099 | BitVector TmpSup(Hexagon::NUM_TARGET_REGS); |
| 2100 | for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(Prev: x)) { |
| 2101 | Register R = x; |
| 2102 | if (Hexagon::HlxLRRegClass.contains(Reg: R)) |
| 2103 | continue; |
| 2104 | for (MCPhysReg SR : TRI->superregs(Reg: R)) |
| 2105 | TmpSup[SR] = true; |
| 2106 | } |
| 2107 | for (int x = TmpSup.find_first(); x >= 0; x = TmpSup.find_next(Prev: x)) { |
| 2108 | Register R = x; |
| 2109 | for (MCPhysReg SR : TRI->subregs_inclusive(Reg: R)) { |
| 2110 | if (!Reserved[SR]) |
| 2111 | continue; |
| 2112 | TmpSup[R] = false; |
| 2113 | break; |
| 2114 | } |
| 2115 | } |
| 2116 | LLVM_DEBUG(dbgs() << "TmpSup: " ; dump_registers(TmpSup, *TRI); |
| 2117 | dbgs() << "\n" ); |
| 2118 | |
| 2119 | // (4) Include all super-registers found in (3) into SRegs. |
| 2120 | SRegs |= TmpSup; |
| 2121 | LLVM_DEBUG(dbgs() << "SRegs.4: " ; dump_registers(SRegs, *TRI); |
| 2122 | dbgs() << "\n" ); |
| 2123 | |
| 2124 | // (5) For each register R in SRegs, if any super-register of R is in SRegs, |
| 2125 | // remove R from SRegs. |
| 2126 | for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(Prev: x)) { |
| 2127 | Register R = x; |
| 2128 | for (MCPhysReg SR : TRI->superregs(Reg: R)) { |
| 2129 | if (!SRegs[SR]) |
| 2130 | continue; |
| 2131 | SRegs[R] = false; |
| 2132 | break; |
| 2133 | } |
| 2134 | } |
| 2135 | LLVM_DEBUG(dbgs() << "SRegs.5: " ; dump_registers(SRegs, *TRI); |
| 2136 | dbgs() << "\n" ); |
| 2137 | |
| 2138 | // Now, for each register that has a fixed stack slot, create the stack |
| 2139 | // object for it. |
| 2140 | CSI.clear(); |
| 2141 | |
| 2142 | using SpillSlot = TargetFrameLowering::SpillSlot; |
| 2143 | |
| 2144 | unsigned NumFixed; |
| 2145 | int64_t MinOffset = 0; // CS offsets are negative. |
| 2146 | const SpillSlot *FixedSlots = getCalleeSavedSpillSlots(NumEntries&: NumFixed); |
| 2147 | for (const SpillSlot *S = FixedSlots; S != FixedSlots+NumFixed; ++S) { |
| 2148 | if (!SRegs[S->Reg]) |
| 2149 | continue; |
| 2150 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: S->Reg); |
| 2151 | int FI = MFI.CreateFixedSpillStackObject(Size: TRI->getSpillSize(RC: *RC), SPOffset: S->Offset); |
| 2152 | MinOffset = std::min(a: MinOffset, b: S->Offset); |
| 2153 | CSI.push_back(x: CalleeSavedInfo(S->Reg, FI)); |
| 2154 | SRegs[S->Reg] = false; |
| 2155 | } |
| 2156 | |
| 2157 | // There can be some registers that don't have fixed slots. For example, |
| 2158 | // we need to store R0-R3 in functions with exception handling. For each |
| 2159 | // such register, create a non-fixed stack object. |
| 2160 | for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(Prev: x)) { |
| 2161 | Register R = x; |
| 2162 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: R); |
| 2163 | unsigned Size = TRI->getSpillSize(RC: *RC); |
| 2164 | // HLX registers are saved in spill slots on the VTCM stack. |
| 2165 | if (Hexagon::HlxLRRegClass.contains(Reg: R)) { |
| 2166 | int FI = MFI.CreateSpillStackObject(Size, Alignment: TRI->getSpillAlign(RC: *RC)); |
| 2167 | CSI.push_back(x: CalleeSavedInfo(R, FI)); |
| 2168 | SRegs[R] = false; |
| 2169 | continue; |
| 2170 | } |
| 2171 | int64_t Off = MinOffset - Size; |
| 2172 | Align Alignment = std::min(a: TRI->getSpillAlign(RC: *RC), b: getStackAlign()); |
| 2173 | Off &= -Alignment.value(); |
| 2174 | int FI = MFI.CreateFixedSpillStackObject(Size, SPOffset: Off); |
| 2175 | MinOffset = std::min(a: MinOffset, b: Off); |
| 2176 | CSI.push_back(x: CalleeSavedInfo(R, FI)); |
| 2177 | SRegs[R] = false; |
| 2178 | } |
| 2179 | |
| 2180 | LLVM_DEBUG({ |
| 2181 | dbgs() << "CS information: {" ; |
| 2182 | for (const CalleeSavedInfo &I : CSI) { |
| 2183 | int FI = I.getFrameIdx(); |
| 2184 | int Off = MFI.getObjectOffset(FI); |
| 2185 | dbgs() << ' ' << printReg(I.getReg(), TRI) << ":fi#" << FI << ":sp" ; |
| 2186 | if (Off >= 0) |
| 2187 | dbgs() << '+'; |
| 2188 | dbgs() << Off; |
| 2189 | } |
| 2190 | dbgs() << " }\n" ; |
| 2191 | }); |
| 2192 | |
| 2193 | #ifndef NDEBUG |
| 2194 | // Verify that all registers were handled. |
| 2195 | bool MissedReg = false; |
| 2196 | for (int x = SRegs.find_first(); x >= 0; x = SRegs.find_next(x)) { |
| 2197 | Register R = x; |
| 2198 | dbgs() << printReg(R, TRI) << ' '; |
| 2199 | MissedReg = true; |
| 2200 | } |
| 2201 | if (MissedReg) |
| 2202 | llvm_unreachable("...there are unhandled callee-saved registers!" ); |
| 2203 | #endif |
| 2204 | |
| 2205 | return true; |
| 2206 | } |
| 2207 | |
| 2208 | bool HexagonFrameLowering::expandCopy(MachineBasicBlock &B, |
| 2209 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2210 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2211 | MachineInstr *MI = &*It; |
| 2212 | DebugLoc DL = MI->getDebugLoc(); |
| 2213 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2214 | Register SrcR = MI->getOperand(i: 1).getReg(); |
| 2215 | if (!Hexagon::ModRegsRegClass.contains(Reg: DstR) || |
| 2216 | !Hexagon::ModRegsRegClass.contains(Reg: SrcR)) |
| 2217 | return false; |
| 2218 | |
| 2219 | Register TmpR = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2220 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: TargetOpcode::COPY), DestReg: TmpR).add(MO: MI->getOperand(i: 1)); |
| 2221 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: TargetOpcode::COPY), DestReg: DstR) |
| 2222 | .addReg(RegNo: TmpR, Flags: RegState::Kill); |
| 2223 | |
| 2224 | NewRegs.push_back(Elt: TmpR); |
| 2225 | B.erase(I: It); |
| 2226 | return true; |
| 2227 | } |
| 2228 | |
| 2229 | bool HexagonFrameLowering::expandStoreInt(MachineBasicBlock &B, |
| 2230 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2231 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2232 | MachineInstr *MI = &*It; |
| 2233 | if (!MI->getOperand(i: 0).isFI()) |
| 2234 | return false; |
| 2235 | |
| 2236 | DebugLoc DL = MI->getDebugLoc(); |
| 2237 | unsigned Opc = MI->getOpcode(); |
| 2238 | Register SrcR = MI->getOperand(i: 2).getReg(); |
| 2239 | bool IsKill = MI->getOperand(i: 2).isKill(); |
| 2240 | int FI = MI->getOperand(i: 0).getIndex(); |
| 2241 | |
| 2242 | // TmpR = C2_tfrpr SrcR if SrcR is a predicate register |
| 2243 | // TmpR = A2_tfrcrr SrcR if SrcR is a modifier register |
| 2244 | Register TmpR = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2245 | unsigned TfrOpc = (Opc == Hexagon::STriw_pred) ? Hexagon::C2_tfrpr |
| 2246 | : Hexagon::A2_tfrcrr; |
| 2247 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: TfrOpc), DestReg: TmpR) |
| 2248 | .addReg(RegNo: SrcR, Flags: getKillRegState(B: IsKill)); |
| 2249 | |
| 2250 | // S2_storeri_io FI, 0, TmpR |
| 2251 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::S2_storeri_io)) |
| 2252 | .addFrameIndex(Idx: FI) |
| 2253 | .addImm(Val: 0) |
| 2254 | .addReg(RegNo: TmpR, Flags: RegState::Kill) |
| 2255 | .cloneMemRefs(OtherMI: *MI); |
| 2256 | |
| 2257 | NewRegs.push_back(Elt: TmpR); |
| 2258 | B.erase(I: It); |
| 2259 | return true; |
| 2260 | } |
| 2261 | |
| 2262 | bool HexagonFrameLowering::expandStoreCirc( |
| 2263 | MachineBasicBlock &B, MachineBasicBlock::iterator It, |
| 2264 | MachineRegisterInfo &MRI, const HexagonInstrInfo &HII, |
| 2265 | SmallVectorImpl<Register> &NewRegs) const { |
| 2266 | MachineFunction &MF = *B.getParent(); |
| 2267 | const auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2268 | MachineInstr *MI = &*It; |
| 2269 | if (!MI->getOperand(i: 0).isFI()) |
| 2270 | return false; |
| 2271 | |
| 2272 | DebugLoc DL = MI->getDebugLoc(); |
| 2273 | Register SrcR = MI->getOperand(i: 1).getReg(); |
| 2274 | bool IsKill = MI->getOperand(i: 1).isKill(); |
| 2275 | int FI = MI->getOperand(i: 0).getIndex(); |
| 2276 | |
| 2277 | unsigned Offset = 0; |
| 2278 | for (auto SubReg : {Hexagon::isub_lo, Hexagon::isub_hi}) { |
| 2279 | Register SrcSubReg = HRI.getSubReg(Reg: SrcR, Idx: SubReg); |
| 2280 | |
| 2281 | // TmpR = A2_tfrcrr SrcSubReg. |
| 2282 | Register TmpR = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2283 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfrcrr), DestReg: TmpR) |
| 2284 | .addReg(RegNo: SrcSubReg, Flags: getKillRegState(B: IsKill)) |
| 2285 | ->copyImplicitOps(MF, MI: *It); |
| 2286 | |
| 2287 | // S2_storeri_io FI/FI+offset, TmpR. |
| 2288 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::S2_storeri_io)) |
| 2289 | .addFrameIndex(Idx: FI) |
| 2290 | .addImm(Val: Offset) |
| 2291 | .addReg(RegNo: TmpR, Flags: RegState::Kill) |
| 2292 | .cloneMemRefs(OtherMI: *MI); |
| 2293 | NewRegs.push_back(Elt: TmpR); |
| 2294 | Offset += HRI.getSpillSize(RC: Hexagon::IntRegsRegClass); |
| 2295 | } |
| 2296 | |
| 2297 | B.erase(I: It); |
| 2298 | return true; |
| 2299 | } |
| 2300 | |
| 2301 | bool HexagonFrameLowering::expandLoadCirc( |
| 2302 | MachineBasicBlock &B, MachineBasicBlock::iterator It, |
| 2303 | MachineRegisterInfo &MRI, const HexagonInstrInfo &HII, |
| 2304 | SmallVectorImpl<Register> &NewRegs) const { |
| 2305 | MachineFunction &MF = *B.getParent(); |
| 2306 | const auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2307 | MachineInstr *MI = &*It; |
| 2308 | if (!MI->getOperand(i: 1).isFI()) |
| 2309 | return false; |
| 2310 | |
| 2311 | DebugLoc DL = MI->getDebugLoc(); |
| 2312 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2313 | int FI = MI->getOperand(i: 1).getIndex(); |
| 2314 | |
| 2315 | unsigned Offset = 0; |
| 2316 | for (auto SubReg : {Hexagon::isub_lo, Hexagon::isub_hi}) { |
| 2317 | Register DstSubReg = HRI.getSubReg(Reg: DstR, Idx: SubReg); |
| 2318 | |
| 2319 | // TmpR = L2_loadri_io FI, Offset. |
| 2320 | Register TmpR = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2321 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::L2_loadri_io), DestReg: TmpR) |
| 2322 | .addFrameIndex(Idx: FI) |
| 2323 | .addImm(Val: Offset) |
| 2324 | .cloneMemRefs(OtherMI: *MI); |
| 2325 | |
| 2326 | // DstR = A2_tfrrcr TmpR. |
| 2327 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfrrcr), DestReg: DstSubReg) |
| 2328 | .addReg(RegNo: TmpR, Flags: RegState::Kill) |
| 2329 | ->copyImplicitOps(MF, MI: *It); |
| 2330 | |
| 2331 | NewRegs.push_back(Elt: TmpR); |
| 2332 | Offset += HRI.getSpillSize(RC: Hexagon::IntRegsRegClass); |
| 2333 | } |
| 2334 | |
| 2335 | B.erase(I: It); |
| 2336 | return true; |
| 2337 | } |
| 2338 | |
| 2339 | bool HexagonFrameLowering::expandLoadInt( |
| 2340 | MachineBasicBlock &B, MachineBasicBlock::iterator It, |
| 2341 | MachineRegisterInfo &MRI, const HexagonInstrInfo &HII, |
| 2342 | SmallVectorImpl<Register> &NewRegs) const { |
| 2343 | MachineInstr *MI = &*It; |
| 2344 | if (!MI->getOperand(i: 1).isFI()) |
| 2345 | return false; |
| 2346 | |
| 2347 | DebugLoc DL = MI->getDebugLoc(); |
| 2348 | unsigned Opc = MI->getOpcode(); |
| 2349 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2350 | int FI = MI->getOperand(i: 1).getIndex(); |
| 2351 | |
| 2352 | // TmpR = L2_loadri_io FI, 0 |
| 2353 | Register TmpR = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2354 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::L2_loadri_io), DestReg: TmpR) |
| 2355 | .addFrameIndex(Idx: FI) |
| 2356 | .addImm(Val: 0) |
| 2357 | .cloneMemRefs(OtherMI: *MI); |
| 2358 | |
| 2359 | // DstR = C2_tfrrp TmpR if DstR is a predicate register |
| 2360 | // DstR = A2_tfrrcr TmpR if DstR is a modifier register |
| 2361 | unsigned TfrOpc = (Opc == Hexagon::LDriw_pred) ? Hexagon::C2_tfrrp |
| 2362 | : Hexagon::A2_tfrrcr; |
| 2363 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: TfrOpc), DestReg: DstR) |
| 2364 | .addReg(RegNo: TmpR, Flags: RegState::Kill); |
| 2365 | |
| 2366 | NewRegs.push_back(Elt: TmpR); |
| 2367 | B.erase(I: It); |
| 2368 | return true; |
| 2369 | } |
| 2370 | |
| 2371 | bool HexagonFrameLowering::expandStoreVecPred(MachineBasicBlock &B, |
| 2372 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2373 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2374 | MachineInstr *MI = &*It; |
| 2375 | if (!MI->getOperand(i: 0).isFI()) |
| 2376 | return false; |
| 2377 | |
| 2378 | DebugLoc DL = MI->getDebugLoc(); |
| 2379 | Register SrcR = MI->getOperand(i: 2).getReg(); |
| 2380 | bool IsKill = MI->getOperand(i: 2).isKill(); |
| 2381 | int FI = MI->getOperand(i: 0).getIndex(); |
| 2382 | auto *RC = &Hexagon::HvxVRRegClass; |
| 2383 | |
| 2384 | // Insert transfer to general vector register. |
| 2385 | // TmpR0 = A2_tfrsi 0x01010101 |
| 2386 | // TmpR1 = V6_vandqrt Qx, TmpR0 |
| 2387 | // store FI, 0, TmpR1 |
| 2388 | Register TmpR0 = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2389 | Register TmpR1 = MRI.createVirtualRegister(RegClass: RC); |
| 2390 | |
| 2391 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfrsi), DestReg: TmpR0) |
| 2392 | .addImm(Val: 0x01010101); |
| 2393 | |
| 2394 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::V6_vandqrt), DestReg: TmpR1) |
| 2395 | .addReg(RegNo: SrcR, Flags: getKillRegState(B: IsKill)) |
| 2396 | .addReg(RegNo: TmpR0, Flags: RegState::Kill); |
| 2397 | |
| 2398 | HII.storeRegToStackSlot(MBB&: B, MBBI: It, SrcReg: TmpR1, isKill: true, FrameIndex: FI, RC, VReg: Register()); |
| 2399 | expandStoreVec(B, It: std::prev(x: It), MRI, HII, NewRegs); |
| 2400 | |
| 2401 | NewRegs.push_back(Elt: TmpR0); |
| 2402 | NewRegs.push_back(Elt: TmpR1); |
| 2403 | B.erase(I: It); |
| 2404 | return true; |
| 2405 | } |
| 2406 | |
| 2407 | bool HexagonFrameLowering::expandLoadVecPred(MachineBasicBlock &B, |
| 2408 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2409 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2410 | MachineInstr *MI = &*It; |
| 2411 | if (!MI->getOperand(i: 1).isFI()) |
| 2412 | return false; |
| 2413 | |
| 2414 | DebugLoc DL = MI->getDebugLoc(); |
| 2415 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2416 | int FI = MI->getOperand(i: 1).getIndex(); |
| 2417 | auto *RC = &Hexagon::HvxVRRegClass; |
| 2418 | |
| 2419 | // TmpR0 = A2_tfrsi 0x01010101 |
| 2420 | // TmpR1 = load FI, 0 |
| 2421 | // DstR = V6_vandvrt TmpR1, TmpR0 |
| 2422 | Register TmpR0 = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2423 | Register TmpR1 = MRI.createVirtualRegister(RegClass: RC); |
| 2424 | |
| 2425 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfrsi), DestReg: TmpR0) |
| 2426 | .addImm(Val: 0x01010101); |
| 2427 | HII.loadRegFromStackSlot(MBB&: B, MBBI: It, DestReg: TmpR1, FrameIndex: FI, RC, VReg: Register()); |
| 2428 | expandLoadVec(B, It: std::prev(x: It), MRI, HII, NewRegs); |
| 2429 | |
| 2430 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::V6_vandvrt), DestReg: DstR) |
| 2431 | .addReg(RegNo: TmpR1, Flags: RegState::Kill) |
| 2432 | .addReg(RegNo: TmpR0, Flags: RegState::Kill); |
| 2433 | |
| 2434 | NewRegs.push_back(Elt: TmpR0); |
| 2435 | NewRegs.push_back(Elt: TmpR1); |
| 2436 | B.erase(I: It); |
| 2437 | return true; |
| 2438 | } |
| 2439 | |
| 2440 | bool HexagonFrameLowering::expandStoreVec2(MachineBasicBlock &B, |
| 2441 | MachineBasicBlock::iterator It, |
| 2442 | MachineRegisterInfo &MRI, |
| 2443 | const HexagonInstrInfo &HII, |
| 2444 | SmallVectorImpl<Register> &NewRegs, |
| 2445 | bool IsHLX) const { |
| 2446 | MachineFunction &MF = *B.getParent(); |
| 2447 | auto &MFI = MF.getFrameInfo(); |
| 2448 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2449 | MachineInstr *MI = &*It; |
| 2450 | if (!MI->getOperand(i: 0).isFI()) |
| 2451 | return false; |
| 2452 | |
| 2453 | // It is possible that the double vector being stored is only partially |
| 2454 | // defined. From the point of view of the liveness tracking, it is ok to |
| 2455 | // store it as a whole, but if we break it up we may end up storing a |
| 2456 | // register that is entirely undefined. |
| 2457 | LivePhysRegs LPR(HRI); |
| 2458 | LPR.addLiveIns(MBB: B); |
| 2459 | SmallVector<std::pair<MCPhysReg, const MachineOperand*>,2> Clobbers; |
| 2460 | for (auto R = B.begin(); R != It; ++R) { |
| 2461 | Clobbers.clear(); |
| 2462 | LPR.stepForward(MI: *R, Clobbers); |
| 2463 | } |
| 2464 | |
| 2465 | DebugLoc DL = MI->getDebugLoc(); |
| 2466 | Register SrcR = MI->getOperand(i: 2).getReg(); |
| 2467 | Register SrcLo = HRI.getSubReg(Reg: SrcR, Idx: Hexagon::vsub_lo); |
| 2468 | Register SrcHi = HRI.getSubReg(Reg: SrcR, Idx: Hexagon::vsub_hi); |
| 2469 | bool IsKill = MI->getOperand(i: 2).isKill(); |
| 2470 | int FI = MI->getOperand(i: 0).getIndex(); |
| 2471 | |
| 2472 | if (IsHLX) { |
| 2473 | const MachineOperand &SrcOp = MI->getOperand(i: 2); |
| 2474 | unsigned Size = HRI.getSpillSize(RC: Hexagon::HlxLRRegClass); |
| 2475 | assert(MI->hasOneMemOperand() && "HLX pair spill must have one MMO" ); |
| 2476 | const MachineMemOperand *PairMMO = *MI->memoperands_begin(); |
| 2477 | MachineMemOperand *LoMMO = MF.getMachineMemOperand(MMO: PairMMO, Offset: 0, Size); |
| 2478 | MachineMemOperand *HiMMO = MF.getMachineMemOperand(MMO: PairMMO, Offset: Size, Size); |
| 2479 | |
| 2480 | if (LPR.contains(Reg: SrcLo)) { |
| 2481 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::LX_S16b_pseudo)) |
| 2482 | .addFrameIndex(Idx: FI) |
| 2483 | .addReg(RegNo: SrcLo, Flags: getRegState(RegOp: SrcOp)) |
| 2484 | .addReg(RegNo: Hexagon::VSP, Flags: RegState::Implicit) |
| 2485 | .addMemOperand(MMO: LoMMO); |
| 2486 | } |
| 2487 | |
| 2488 | if (LPR.contains(Reg: SrcHi)) { |
| 2489 | Register StackReg = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2490 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: StackReg) |
| 2491 | .addFrameIndex(Idx: FI) |
| 2492 | .addImm(Val: Size); |
| 2493 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::LX_S16b_pseudo)) |
| 2494 | .addReg(RegNo: StackReg, Flags: RegState::Kill) |
| 2495 | .addReg(RegNo: SrcHi, Flags: getRegState(RegOp: SrcOp)) |
| 2496 | .addReg(RegNo: Hexagon::VSP, Flags: RegState::Implicit) |
| 2497 | .addMemOperand(MMO: HiMMO); |
| 2498 | } |
| 2499 | |
| 2500 | B.erase(I: It); |
| 2501 | return true; |
| 2502 | } |
| 2503 | |
| 2504 | unsigned Size = HRI.getSpillSize(RC: Hexagon::HvxVRRegClass); |
| 2505 | Align NeedAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass); |
| 2506 | Align HasAlign = MFI.getObjectAlign(ObjectIdx: FI); |
| 2507 | unsigned StoreOpc; |
| 2508 | |
| 2509 | // Store low part. |
| 2510 | if (LPR.contains(Reg: SrcLo)) { |
| 2511 | StoreOpc = NeedAlign <= HasAlign ? Hexagon::V6_vS32b_ai |
| 2512 | : Hexagon::V6_vS32Ub_ai; |
| 2513 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: StoreOpc)) |
| 2514 | .addFrameIndex(Idx: FI) |
| 2515 | .addImm(Val: 0) |
| 2516 | .addReg(RegNo: SrcLo, Flags: getKillRegState(B: IsKill)) |
| 2517 | .cloneMemRefs(OtherMI: *MI); |
| 2518 | } |
| 2519 | |
| 2520 | // Store high part. |
| 2521 | if (LPR.contains(Reg: SrcHi)) { |
| 2522 | StoreOpc = NeedAlign <= HasAlign ? Hexagon::V6_vS32b_ai |
| 2523 | : Hexagon::V6_vS32Ub_ai; |
| 2524 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: StoreOpc)) |
| 2525 | .addFrameIndex(Idx: FI) |
| 2526 | .addImm(Val: Size) |
| 2527 | .addReg(RegNo: SrcHi, Flags: getKillRegState(B: IsKill)) |
| 2528 | .cloneMemRefs(OtherMI: *MI); |
| 2529 | } |
| 2530 | |
| 2531 | B.erase(I: It); |
| 2532 | return true; |
| 2533 | } |
| 2534 | |
| 2535 | bool HexagonFrameLowering::expandLoadVec2(MachineBasicBlock &B, |
| 2536 | MachineBasicBlock::iterator It, |
| 2537 | MachineRegisterInfo &MRI, |
| 2538 | const HexagonInstrInfo &HII, |
| 2539 | SmallVectorImpl<Register> &NewRegs, |
| 2540 | bool IsHLX) const { |
| 2541 | MachineFunction &MF = *B.getParent(); |
| 2542 | auto &MFI = MF.getFrameInfo(); |
| 2543 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2544 | MachineInstr *MI = &*It; |
| 2545 | if (!MI->getOperand(i: 1).isFI()) |
| 2546 | return false; |
| 2547 | |
| 2548 | DebugLoc DL = MI->getDebugLoc(); |
| 2549 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2550 | Register DstHi = HRI.getSubReg(Reg: DstR, Idx: Hexagon::vsub_hi); |
| 2551 | Register DstLo = HRI.getSubReg(Reg: DstR, Idx: Hexagon::vsub_lo); |
| 2552 | int FI = MI->getOperand(i: 1).getIndex(); |
| 2553 | |
| 2554 | if (IsHLX) { |
| 2555 | const MachineOperand &DstOp = MI->getOperand(i: 0); |
| 2556 | unsigned Size = HRI.getSpillSize(RC: Hexagon::HlxLRRegClass); |
| 2557 | assert(MI->hasOneMemOperand() && "HLX pair reload must have one MMO" ); |
| 2558 | const MachineMemOperand *PairMMO = *MI->memoperands_begin(); |
| 2559 | MachineMemOperand *LoMMO = MF.getMachineMemOperand(MMO: PairMMO, Offset: 0, Size); |
| 2560 | MachineMemOperand *HiMMO = MF.getMachineMemOperand(MMO: PairMMO, Offset: Size, Size); |
| 2561 | |
| 2562 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::LX_L16b_pseudo)) |
| 2563 | .addReg(RegNo: DstLo, Flags: getRegState(RegOp: DstOp)) |
| 2564 | .addFrameIndex(Idx: FI) |
| 2565 | .addReg(RegNo: Hexagon::VSP, Flags: RegState::Implicit) |
| 2566 | .addMemOperand(MMO: LoMMO); |
| 2567 | |
| 2568 | Register StackReg = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass); |
| 2569 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: StackReg) |
| 2570 | .addFrameIndex(Idx: FI) |
| 2571 | .addImm(Val: Size); |
| 2572 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: Hexagon::LX_L16b_pseudo)) |
| 2573 | .addReg(RegNo: DstHi, Flags: getRegState(RegOp: DstOp)) |
| 2574 | .addReg(RegNo: StackReg, Flags: RegState::Kill) |
| 2575 | .addReg(RegNo: Hexagon::VSP, Flags: RegState::Implicit) |
| 2576 | .addMemOperand(MMO: HiMMO); |
| 2577 | |
| 2578 | B.erase(I: It); |
| 2579 | return true; |
| 2580 | } |
| 2581 | |
| 2582 | unsigned Size = HRI.getSpillSize(RC: Hexagon::HvxVRRegClass); |
| 2583 | Align NeedAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass); |
| 2584 | Align HasAlign = MFI.getObjectAlign(ObjectIdx: FI); |
| 2585 | unsigned LoadOpc; |
| 2586 | |
| 2587 | // Load low part. |
| 2588 | LoadOpc = NeedAlign <= HasAlign ? Hexagon::V6_vL32b_ai |
| 2589 | : Hexagon::V6_vL32Ub_ai; |
| 2590 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: LoadOpc), DestReg: DstLo) |
| 2591 | .addFrameIndex(Idx: FI) |
| 2592 | .addImm(Val: 0) |
| 2593 | .cloneMemRefs(OtherMI: *MI); |
| 2594 | |
| 2595 | // Load high part. |
| 2596 | LoadOpc = NeedAlign <= HasAlign ? Hexagon::V6_vL32b_ai |
| 2597 | : Hexagon::V6_vL32Ub_ai; |
| 2598 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: LoadOpc), DestReg: DstHi) |
| 2599 | .addFrameIndex(Idx: FI) |
| 2600 | .addImm(Val: Size) |
| 2601 | .cloneMemRefs(OtherMI: *MI); |
| 2602 | |
| 2603 | B.erase(I: It); |
| 2604 | return true; |
| 2605 | } |
| 2606 | |
| 2607 | bool HexagonFrameLowering::expandStoreVec(MachineBasicBlock &B, |
| 2608 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2609 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2610 | MachineFunction &MF = *B.getParent(); |
| 2611 | auto &MFI = MF.getFrameInfo(); |
| 2612 | MachineInstr *MI = &*It; |
| 2613 | if (!MI->getOperand(i: 0).isFI()) |
| 2614 | return false; |
| 2615 | |
| 2616 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2617 | DebugLoc DL = MI->getDebugLoc(); |
| 2618 | Register SrcR = MI->getOperand(i: 2).getReg(); |
| 2619 | bool IsKill = MI->getOperand(i: 2).isKill(); |
| 2620 | int FI = MI->getOperand(i: 0).getIndex(); |
| 2621 | |
| 2622 | Align NeedAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass); |
| 2623 | Align HasAlign = MFI.getObjectAlign(ObjectIdx: FI); |
| 2624 | unsigned StoreOpc = NeedAlign <= HasAlign ? Hexagon::V6_vS32b_ai |
| 2625 | : Hexagon::V6_vS32Ub_ai; |
| 2626 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: StoreOpc)) |
| 2627 | .addFrameIndex(Idx: FI) |
| 2628 | .addImm(Val: 0) |
| 2629 | .addReg(RegNo: SrcR, Flags: getKillRegState(B: IsKill)) |
| 2630 | .cloneMemRefs(OtherMI: *MI); |
| 2631 | |
| 2632 | B.erase(I: It); |
| 2633 | return true; |
| 2634 | } |
| 2635 | |
| 2636 | bool HexagonFrameLowering::expandLoadVec(MachineBasicBlock &B, |
| 2637 | MachineBasicBlock::iterator It, MachineRegisterInfo &MRI, |
| 2638 | const HexagonInstrInfo &HII, SmallVectorImpl<Register> &NewRegs) const { |
| 2639 | MachineFunction &MF = *B.getParent(); |
| 2640 | auto &MFI = MF.getFrameInfo(); |
| 2641 | MachineInstr *MI = &*It; |
| 2642 | if (!MI->getOperand(i: 1).isFI()) |
| 2643 | return false; |
| 2644 | |
| 2645 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2646 | DebugLoc DL = MI->getDebugLoc(); |
| 2647 | Register DstR = MI->getOperand(i: 0).getReg(); |
| 2648 | int FI = MI->getOperand(i: 1).getIndex(); |
| 2649 | |
| 2650 | Align NeedAlign = HRI.getSpillAlign(RC: Hexagon::HvxVRRegClass); |
| 2651 | Align HasAlign = MFI.getObjectAlign(ObjectIdx: FI); |
| 2652 | unsigned LoadOpc = NeedAlign <= HasAlign ? Hexagon::V6_vL32b_ai |
| 2653 | : Hexagon::V6_vL32Ub_ai; |
| 2654 | BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: LoadOpc), DestReg: DstR) |
| 2655 | .addFrameIndex(Idx: FI) |
| 2656 | .addImm(Val: 0) |
| 2657 | .cloneMemRefs(OtherMI: *MI); |
| 2658 | |
| 2659 | B.erase(I: It); |
| 2660 | return true; |
| 2661 | } |
| 2662 | |
| 2663 | bool HexagonFrameLowering::expandSpillMacros(MachineFunction &MF, |
| 2664 | SmallVectorImpl<Register> &NewRegs) const { |
| 2665 | auto &HII = *MF.getSubtarget<HexagonSubtarget>().getInstrInfo(); |
| 2666 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2667 | bool Changed = false; |
| 2668 | |
| 2669 | for (auto &B : MF) { |
| 2670 | // Traverse the basic block. |
| 2671 | MachineBasicBlock::iterator NextI; |
| 2672 | for (auto I = B.begin(), E = B.end(); I != E; I = NextI) { |
| 2673 | MachineInstr *MI = &*I; |
| 2674 | NextI = std::next(x: I); |
| 2675 | unsigned Opc = MI->getOpcode(); |
| 2676 | |
| 2677 | switch (Opc) { |
| 2678 | case TargetOpcode::COPY: |
| 2679 | Changed |= expandCopy(B, It: I, MRI, HII, NewRegs); |
| 2680 | break; |
| 2681 | case Hexagon::STriw_pred: |
| 2682 | case Hexagon::STriw_ctr: |
| 2683 | Changed |= expandStoreInt(B, It: I, MRI, HII, NewRegs); |
| 2684 | break; |
| 2685 | case Hexagon::STriw_circ: |
| 2686 | Changed |= expandStoreCirc(B, It: I, MRI, HII, NewRegs); |
| 2687 | break; |
| 2688 | case Hexagon::LDriw_pred: |
| 2689 | case Hexagon::LDriw_ctr: |
| 2690 | Changed |= expandLoadInt(B, It: I, MRI, HII, NewRegs); |
| 2691 | break; |
| 2692 | case Hexagon::LDriw_circ: |
| 2693 | Changed |= expandLoadCirc(B, It: I, MRI, HII, NewRegs); |
| 2694 | break; |
| 2695 | case Hexagon::PS_vstorerq_ai: |
| 2696 | Changed |= expandStoreVecPred(B, It: I, MRI, HII, NewRegs); |
| 2697 | break; |
| 2698 | case Hexagon::PS_vloadrq_ai: |
| 2699 | Changed |= expandLoadVecPred(B, It: I, MRI, HII, NewRegs); |
| 2700 | break; |
| 2701 | case Hexagon::PS_vloadrw_ai: |
| 2702 | Changed |= expandLoadVec2(B, It: I, MRI, HII, NewRegs); |
| 2703 | break; |
| 2704 | case Hexagon::PS_vstorerw_ai: |
| 2705 | Changed |= expandStoreVec2(B, It: I, MRI, HII, NewRegs); |
| 2706 | break; |
| 2707 | case Hexagon::PS_loadrll: |
| 2708 | Changed |= expandLoadVec2(B, It: I, MRI, HII, NewRegs, /*IsHLX=*/true); |
| 2709 | break; |
| 2710 | case Hexagon::PS_storerll: |
| 2711 | Changed |= expandStoreVec2(B, It: I, MRI, HII, NewRegs, /*IsHLX=*/true); |
| 2712 | break; |
| 2713 | } |
| 2714 | } |
| 2715 | } |
| 2716 | |
| 2717 | return Changed; |
| 2718 | } |
| 2719 | |
| 2720 | void HexagonFrameLowering::determineCalleeSaves(MachineFunction &MF, |
| 2721 | BitVector &SavedRegs, |
| 2722 | RegScavenger *RS) const { |
| 2723 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2724 | |
| 2725 | SavedRegs.resize(N: HRI.getNumRegs()); |
| 2726 | |
| 2727 | // If we have a function containing __builtin_eh_return we want to spill and |
| 2728 | // restore all callee saved registers. Pretend that they are used. |
| 2729 | if (MF.getInfo<HexagonMachineFunctionInfo>()->hasEHReturn()) |
| 2730 | for (const MCPhysReg *R = HRI.getCalleeSavedRegs(MF: &MF); *R; ++R) |
| 2731 | SavedRegs.set(*R); |
| 2732 | |
| 2733 | // If the function needs dynamic stack realignment, AP is a callee-saved |
| 2734 | // register that gets overwritten by the PS_aligna emitted in the prologue |
| 2735 | // but PS_aligna is created during emitPrologue, which runs after this hook. |
| 2736 | if (needsAligna(MF)) { |
| 2737 | Register AP = |
| 2738 | MF.getInfo<HexagonMachineFunctionInfo>()->getStackAlignBaseReg(); |
| 2739 | assert(AP.isValid() && "AP must be assigned before register allocation" ); |
| 2740 | SavedRegs.set(AP); |
| 2741 | } |
| 2742 | |
| 2743 | // Replace predicate register pseudo spill code. |
| 2744 | SmallVector<Register,8> NewRegs; |
| 2745 | expandSpillMacros(MF, NewRegs); |
| 2746 | if (OptimizeSpillSlots && !isOptNone(MF)) |
| 2747 | optimizeSpillSlots(MF, VRegs&: NewRegs); |
| 2748 | |
| 2749 | // We need to reserve a spill slot if scavenging could potentially require |
| 2750 | // spilling a scavenged register. |
| 2751 | if (!NewRegs.empty() || mayOverflowFrameOffset(MF)) { |
| 2752 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 2753 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2754 | SetVector<const TargetRegisterClass*> SpillRCs; |
| 2755 | // Reserve an int register in any case, because it could be used to hold |
| 2756 | // the stack offset in case it does not fit into a spill instruction. |
| 2757 | SpillRCs.insert(X: &Hexagon::IntRegsRegClass); |
| 2758 | |
| 2759 | for (Register VR : NewRegs) |
| 2760 | SpillRCs.insert(X: MRI.getRegClass(Reg: VR)); |
| 2761 | |
| 2762 | for (const auto *RC : SpillRCs) { |
| 2763 | if (!needToReserveScavengingSpillSlots(MF, HRI, RC)) |
| 2764 | continue; |
| 2765 | unsigned Num = 1; |
| 2766 | switch (RC->getID()) { |
| 2767 | case Hexagon::IntRegsRegClassID: |
| 2768 | Num = NumberScavengerSlots; |
| 2769 | break; |
| 2770 | case Hexagon::HvxQRRegClassID: |
| 2771 | Num = 2; // Vector predicate spills also need a vector register. |
| 2772 | break; |
| 2773 | } |
| 2774 | unsigned S = HRI.getSpillSize(RC: *RC); |
| 2775 | Align A = HRI.getSpillAlign(RC: *RC); |
| 2776 | for (unsigned i = 0; i < Num; i++) { |
| 2777 | int NewFI = MFI.CreateSpillStackObject(Size: S, Alignment: A); |
| 2778 | RS->addScavengingFrameIndex(FI: NewFI); |
| 2779 | } |
| 2780 | } |
| 2781 | } |
| 2782 | |
| 2783 | TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); |
| 2784 | } |
| 2785 | |
| 2786 | Register HexagonFrameLowering::findPhysReg(MachineFunction &MF, |
| 2787 | HexagonBlockRanges::IndexRange &FIR, |
| 2788 | HexagonBlockRanges::InstrIndexMap &IndexMap, |
| 2789 | HexagonBlockRanges::RegToRangeMap &DeadMap, |
| 2790 | const TargetRegisterClass *RC) const { |
| 2791 | auto &HRI = *MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
| 2792 | auto &MRI = MF.getRegInfo(); |
| 2793 | |
| 2794 | auto isDead = [&FIR,&DeadMap] (Register Reg) -> bool { |
| 2795 | auto F = DeadMap.find(x: {.Reg: Reg,.Sub: 0}); |
| 2796 | if (F == DeadMap.end()) |
| 2797 | return false; |
| 2798 | for (auto &DR : F->second) |
| 2799 | if (DR.contains(A: FIR)) |
| 2800 | return true; |
| 2801 | return false; |
| 2802 | }; |
| 2803 | |
| 2804 | for (Register Reg : HRI.getRawAllocationOrder(RC: *RC, MF)) { |
| 2805 | bool Dead = true; |
| 2806 | for (auto R : HexagonBlockRanges::expandToSubRegs(R: {.Reg: Reg,.Sub: 0}, MRI, TRI: HRI)) { |
| 2807 | if (isDead(R.Reg)) |
| 2808 | continue; |
| 2809 | Dead = false; |
| 2810 | break; |
| 2811 | } |
| 2812 | if (Dead) |
| 2813 | return Reg; |
| 2814 | } |
| 2815 | return 0; |
| 2816 | } |
| 2817 | |
| 2818 | void HexagonFrameLowering::optimizeSpillSlots(MachineFunction &MF, |
| 2819 | SmallVectorImpl<Register> &VRegs) const { |
| 2820 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 2821 | auto &HII = *HST.getInstrInfo(); |
| 2822 | auto &HRI = *HST.getRegisterInfo(); |
| 2823 | auto &MRI = MF.getRegInfo(); |
| 2824 | HexagonBlockRanges HBR(MF); |
| 2825 | |
| 2826 | using BlockIndexMap = |
| 2827 | std::map<MachineBasicBlock *, HexagonBlockRanges::InstrIndexMap>; |
| 2828 | using BlockRangeMap = |
| 2829 | std::map<MachineBasicBlock *, HexagonBlockRanges::RangeList>; |
| 2830 | using IndexType = HexagonBlockRanges::IndexType; |
| 2831 | |
| 2832 | struct SlotInfo { |
| 2833 | BlockRangeMap Map; |
| 2834 | unsigned Size = 0; |
| 2835 | const TargetRegisterClass *RC = nullptr; |
| 2836 | |
| 2837 | SlotInfo() = default; |
| 2838 | }; |
| 2839 | |
| 2840 | BlockIndexMap BlockIndexes; |
| 2841 | SmallSet<int,4> BadFIs; |
| 2842 | std::map<int,SlotInfo> FIRangeMap; |
| 2843 | |
| 2844 | // Accumulate register classes: get a common class for a pre-existing |
| 2845 | // class HaveRC and a new class NewRC. Return nullptr if a common class |
| 2846 | // cannot be found, otherwise return the resulting class. If HaveRC is |
| 2847 | // nullptr, assume that it is still unset. |
| 2848 | auto getCommonRC = |
| 2849 | [](const TargetRegisterClass *HaveRC, |
| 2850 | const TargetRegisterClass *NewRC) -> const TargetRegisterClass * { |
| 2851 | if (HaveRC == nullptr || HaveRC == NewRC) |
| 2852 | return NewRC; |
| 2853 | // Different classes, both non-null. Pick the more general one. |
| 2854 | if (HaveRC->hasSubClassEq(RC: NewRC)) |
| 2855 | return HaveRC; |
| 2856 | if (NewRC->hasSubClassEq(RC: HaveRC)) |
| 2857 | return NewRC; |
| 2858 | return nullptr; |
| 2859 | }; |
| 2860 | |
| 2861 | // Scan all blocks in the function. Check all occurrences of frame indexes, |
| 2862 | // and collect relevant information. |
| 2863 | for (auto &B : MF) { |
| 2864 | std::map<int,IndexType> LastStore, LastLoad; |
| 2865 | auto P = BlockIndexes.emplace(args: &B, args: HexagonBlockRanges::InstrIndexMap(B)); |
| 2866 | auto &IndexMap = P.first->second; |
| 2867 | LLVM_DEBUG(dbgs() << "Index map for " << printMBBReference(B) << "\n" |
| 2868 | << IndexMap << '\n'); |
| 2869 | |
| 2870 | for (auto &In : B) { |
| 2871 | // Debug instructions do not generate any code, and their operands |
| 2872 | // (including frame index operands) must not affect the decisions made |
| 2873 | // by this optimization. |
| 2874 | if (In.isDebugInstr()) |
| 2875 | continue; |
| 2876 | int LFI, SFI; |
| 2877 | bool Load = HII.isLoadFromStackSlot(MI: In, FrameIndex&: LFI) && !HII.isPredicated(MI: In); |
| 2878 | bool Store = HII.isStoreToStackSlot(MI: In, FrameIndex&: SFI) && !HII.isPredicated(MI: In); |
| 2879 | if (Load && Store) { |
| 2880 | // If it's both a load and a store, then we won't handle it. |
| 2881 | BadFIs.insert(V: LFI); |
| 2882 | BadFIs.insert(V: SFI); |
| 2883 | continue; |
| 2884 | } |
| 2885 | // Check for register classes of the register used as the source for |
| 2886 | // the store, and the register used as the destination for the load. |
| 2887 | // Also, only accept base+imm_offset addressing modes. Other addressing |
| 2888 | // modes can have side-effects (post-increments, etc.). For stack |
| 2889 | // slots they are very unlikely, so there is not much loss due to |
| 2890 | // this restriction. |
| 2891 | if (Load || Store) { |
| 2892 | int TFI = Load ? LFI : SFI; |
| 2893 | unsigned AM = HII.getAddrMode(MI: In); |
| 2894 | SlotInfo &SI = FIRangeMap[TFI]; |
| 2895 | bool Bad = (AM != HexagonII::BaseImmOffset); |
| 2896 | if (!Bad) { |
| 2897 | // If the addressing mode is ok, check the register class. |
| 2898 | unsigned OpNum = Load ? 0 : 2; |
| 2899 | auto *RC = HII.getRegClass(MCID: In.getDesc(), OpNum); |
| 2900 | RC = getCommonRC(SI.RC, RC); |
| 2901 | if (RC == nullptr) |
| 2902 | Bad = true; |
| 2903 | else |
| 2904 | SI.RC = RC; |
| 2905 | } |
| 2906 | if (!Bad) { |
| 2907 | // Check sizes. |
| 2908 | unsigned S = HII.getMemAccessSize(MI: In); |
| 2909 | if (SI.Size != 0 && SI.Size != S) |
| 2910 | Bad = true; |
| 2911 | else |
| 2912 | SI.Size = S; |
| 2913 | } |
| 2914 | if (!Bad) { |
| 2915 | for (auto *Mo : In.memoperands()) { |
| 2916 | if (!Mo->isVolatile() && !Mo->isAtomic()) |
| 2917 | continue; |
| 2918 | Bad = true; |
| 2919 | break; |
| 2920 | } |
| 2921 | } |
| 2922 | if (Bad) |
| 2923 | BadFIs.insert(V: TFI); |
| 2924 | } |
| 2925 | |
| 2926 | // Locate uses of frame indices. |
| 2927 | for (unsigned i = 0, n = In.getNumOperands(); i < n; ++i) { |
| 2928 | const MachineOperand &Op = In.getOperand(i); |
| 2929 | if (!Op.isFI()) |
| 2930 | continue; |
| 2931 | int FI = Op.getIndex(); |
| 2932 | // Make sure that the following operand is an immediate and that |
| 2933 | // it is 0. This is the offset in the stack object. |
| 2934 | if (i+1 >= n || !In.getOperand(i: i+1).isImm() || |
| 2935 | In.getOperand(i: i+1).getImm() != 0) |
| 2936 | BadFIs.insert(V: FI); |
| 2937 | if (BadFIs.count(V: FI)) |
| 2938 | continue; |
| 2939 | |
| 2940 | IndexType Index = IndexMap.getIndex(MI: &In); |
| 2941 | auto &LS = LastStore[FI]; |
| 2942 | auto &LL = LastLoad[FI]; |
| 2943 | if (Load) { |
| 2944 | if (LS == IndexType::None) |
| 2945 | LS = IndexType::Entry; |
| 2946 | LL = Index; |
| 2947 | } else if (Store) { |
| 2948 | HexagonBlockRanges::RangeList &RL = FIRangeMap[FI].Map[&B]; |
| 2949 | if (LS != IndexType::None) |
| 2950 | RL.add(Start: LS, End: LL, Fixed: false, TiedEnd: false); |
| 2951 | else if (LL != IndexType::None) |
| 2952 | RL.add(Start: IndexType::Entry, End: LL, Fixed: false, TiedEnd: false); |
| 2953 | LL = IndexType::None; |
| 2954 | LS = Index; |
| 2955 | } else { |
| 2956 | BadFIs.insert(V: FI); |
| 2957 | } |
| 2958 | } |
| 2959 | } |
| 2960 | |
| 2961 | for (auto &I : LastLoad) { |
| 2962 | IndexType LL = I.second; |
| 2963 | if (LL == IndexType::None) |
| 2964 | continue; |
| 2965 | auto &RL = FIRangeMap[I.first].Map[&B]; |
| 2966 | IndexType &LS = LastStore[I.first]; |
| 2967 | if (LS != IndexType::None) |
| 2968 | RL.add(Start: LS, End: LL, Fixed: false, TiedEnd: false); |
| 2969 | else |
| 2970 | RL.add(Start: IndexType::Entry, End: LL, Fixed: false, TiedEnd: false); |
| 2971 | LS = IndexType::None; |
| 2972 | } |
| 2973 | for (auto &I : LastStore) { |
| 2974 | IndexType LS = I.second; |
| 2975 | if (LS == IndexType::None) |
| 2976 | continue; |
| 2977 | auto &RL = FIRangeMap[I.first].Map[&B]; |
| 2978 | RL.add(Start: LS, End: IndexType::None, Fixed: false, TiedEnd: false); |
| 2979 | } |
| 2980 | } |
| 2981 | |
| 2982 | LLVM_DEBUG({ |
| 2983 | for (auto &P : FIRangeMap) { |
| 2984 | dbgs() << "fi#" << P.first; |
| 2985 | if (BadFIs.count(P.first)) |
| 2986 | dbgs() << " (bad)" ; |
| 2987 | dbgs() << " RC: " ; |
| 2988 | if (P.second.RC != nullptr) |
| 2989 | dbgs() << HRI.getRegClassName(P.second.RC) << '\n'; |
| 2990 | else |
| 2991 | dbgs() << "<null>\n" ; |
| 2992 | for (auto &R : P.second.Map) |
| 2993 | dbgs() << " " << printMBBReference(*R.first) << " { " << R.second |
| 2994 | << "}\n" ; |
| 2995 | } |
| 2996 | }); |
| 2997 | |
| 2998 | // When a slot is loaded from in a block without being stored to in the |
| 2999 | // same block, it is live-on-entry to this block. To avoid CFG analysis, |
| 3000 | // consider this slot to be live-on-exit from all blocks. |
| 3001 | SmallSet<int,4> LoxFIs; |
| 3002 | |
| 3003 | std::map<MachineBasicBlock*,std::vector<int>> BlockFIMap; |
| 3004 | |
| 3005 | for (auto &P : FIRangeMap) { |
| 3006 | // P = pair(FI, map: BB->RangeList) |
| 3007 | if (BadFIs.count(V: P.first)) |
| 3008 | continue; |
| 3009 | for (auto &B : MF) { |
| 3010 | auto F = P.second.Map.find(x: &B); |
| 3011 | // F = pair(BB, RangeList) |
| 3012 | if (F == P.second.Map.end() || F->second.empty()) |
| 3013 | continue; |
| 3014 | HexagonBlockRanges::IndexRange &IR = F->second.front(); |
| 3015 | if (IR.start() == IndexType::Entry) |
| 3016 | LoxFIs.insert(V: P.first); |
| 3017 | BlockFIMap[&B].push_back(x: P.first); |
| 3018 | } |
| 3019 | } |
| 3020 | |
| 3021 | LLVM_DEBUG({ |
| 3022 | dbgs() << "Block-to-FI map (* -- live-on-exit):\n" ; |
| 3023 | for (auto &P : BlockFIMap) { |
| 3024 | auto &FIs = P.second; |
| 3025 | if (FIs.empty()) |
| 3026 | continue; |
| 3027 | dbgs() << " " << printMBBReference(*P.first) << ": {" ; |
| 3028 | for (auto I : FIs) { |
| 3029 | dbgs() << " fi#" << I; |
| 3030 | if (LoxFIs.count(I)) |
| 3031 | dbgs() << '*'; |
| 3032 | } |
| 3033 | dbgs() << " }\n" ; |
| 3034 | } |
| 3035 | }); |
| 3036 | |
| 3037 | #ifndef NDEBUG |
| 3038 | bool HasOptLimit = SpillOptMax.getPosition(); |
| 3039 | #endif |
| 3040 | |
| 3041 | // eliminate loads, when all loads eliminated, eliminate all stores. |
| 3042 | for (auto &B : MF) { |
| 3043 | auto F = BlockIndexes.find(x: &B); |
| 3044 | assert(F != BlockIndexes.end()); |
| 3045 | HexagonBlockRanges::InstrIndexMap &IM = F->second; |
| 3046 | HexagonBlockRanges::RegToRangeMap LM = HBR.computeLiveMap(IndexMap&: IM); |
| 3047 | HexagonBlockRanges::RegToRangeMap DM = HBR.computeDeadMap(IndexMap&: IM, LiveMap&: LM); |
| 3048 | LLVM_DEBUG(dbgs() << printMBBReference(B) << " dead map\n" |
| 3049 | << HexagonBlockRanges::PrintRangeMap(DM, HRI)); |
| 3050 | |
| 3051 | for (auto FI : BlockFIMap[&B]) { |
| 3052 | if (BadFIs.count(V: FI)) |
| 3053 | continue; |
| 3054 | LLVM_DEBUG(dbgs() << "Working on fi#" << FI << '\n'); |
| 3055 | HexagonBlockRanges::RangeList &RL = FIRangeMap[FI].Map[&B]; |
| 3056 | for (auto &Range : RL) { |
| 3057 | LLVM_DEBUG(dbgs() << "--Examining range:" << RL << '\n'); |
| 3058 | if (!IndexType::isInstr(X: Range.start()) || |
| 3059 | !IndexType::isInstr(X: Range.end())) |
| 3060 | continue; |
| 3061 | MachineInstr &SI = *IM.getInstr(Idx: Range.start()); |
| 3062 | MachineInstr &EI = *IM.getInstr(Idx: Range.end()); |
| 3063 | assert(SI.mayStore() && "Unexpected start instruction" ); |
| 3064 | assert(EI.mayLoad() && "Unexpected end instruction" ); |
| 3065 | MachineOperand &SrcOp = SI.getOperand(i: 2); |
| 3066 | |
| 3067 | HexagonBlockRanges::RegisterRef SrcRR = { .Reg: SrcOp.getReg(), |
| 3068 | .Sub: SrcOp.getSubReg() }; |
| 3069 | auto *RC = HII.getRegClass(MCID: SI.getDesc(), OpNum: 2); |
| 3070 | // The this-> is needed to unconfuse MSVC. |
| 3071 | Register FoundR = this->findPhysReg(MF, FIR&: Range, IndexMap&: IM, DeadMap&: DM, RC); |
| 3072 | LLVM_DEBUG(dbgs() << "Replacement reg:" << printReg(FoundR, &HRI) |
| 3073 | << '\n'); |
| 3074 | if (FoundR == 0) |
| 3075 | continue; |
| 3076 | #ifndef NDEBUG |
| 3077 | if (HasOptLimit) { |
| 3078 | if (SpillOptCount >= SpillOptMax) |
| 3079 | return; |
| 3080 | SpillOptCount++; |
| 3081 | } |
| 3082 | #endif |
| 3083 | |
| 3084 | // Generate the copy-in: "FoundR = COPY SrcR" at the store location. |
| 3085 | MachineBasicBlock::iterator StartIt = SI.getIterator(), NextIt; |
| 3086 | MachineInstr *CopyIn = nullptr; |
| 3087 | if (SrcRR.Reg != FoundR || SrcRR.Sub != 0) { |
| 3088 | const DebugLoc &DL = SI.getDebugLoc(); |
| 3089 | CopyIn = BuildMI(BB&: B, I: StartIt, MIMD: DL, MCID: HII.get(Opcode: TargetOpcode::COPY), DestReg: FoundR) |
| 3090 | .add(MO: SrcOp); |
| 3091 | } |
| 3092 | |
| 3093 | ++StartIt; |
| 3094 | // Check if this is a last store and the FI is live-on-exit. |
| 3095 | if (LoxFIs.count(V: FI) && (&Range == &RL.back())) { |
| 3096 | // Update store's source register. |
| 3097 | if (unsigned SR = SrcOp.getSubReg()) |
| 3098 | SrcOp.setReg(HRI.getSubReg(Reg: FoundR, Idx: SR)); |
| 3099 | else |
| 3100 | SrcOp.setReg(FoundR); |
| 3101 | SrcOp.setSubReg(0); |
| 3102 | // We are keeping this register live. |
| 3103 | SrcOp.setIsKill(false); |
| 3104 | } else { |
| 3105 | B.erase(I: &SI); |
| 3106 | IM.replaceInstr(OldMI: &SI, NewMI: CopyIn); |
| 3107 | } |
| 3108 | |
| 3109 | auto EndIt = std::next(x: EI.getIterator()); |
| 3110 | for (auto It = StartIt; It != EndIt; It = NextIt) { |
| 3111 | MachineInstr &MI = *It; |
| 3112 | NextIt = std::next(x: It); |
| 3113 | int TFI; |
| 3114 | if (!HII.isLoadFromStackSlot(MI, FrameIndex&: TFI) || TFI != FI) |
| 3115 | continue; |
| 3116 | Register DstR = MI.getOperand(i: 0).getReg(); |
| 3117 | assert(MI.getOperand(0).getSubReg() == 0); |
| 3118 | MachineInstr *CopyOut = nullptr; |
| 3119 | if (DstR != FoundR) { |
| 3120 | DebugLoc DL = MI.getDebugLoc(); |
| 3121 | unsigned MemSize = HII.getMemAccessSize(MI); |
| 3122 | assert(HII.getAddrMode(MI) == HexagonII::BaseImmOffset); |
| 3123 | unsigned CopyOpc = TargetOpcode::COPY; |
| 3124 | if (HII.isSignExtendingLoad(MI)) |
| 3125 | CopyOpc = (MemSize == 1) ? Hexagon::A2_sxtb : Hexagon::A2_sxth; |
| 3126 | else if (HII.isZeroExtendingLoad(MI)) |
| 3127 | CopyOpc = (MemSize == 1) ? Hexagon::A2_zxtb : Hexagon::A2_zxth; |
| 3128 | CopyOut = BuildMI(BB&: B, I: It, MIMD: DL, MCID: HII.get(Opcode: CopyOpc), DestReg: DstR) |
| 3129 | .addReg(RegNo: FoundR, Flags: getKillRegState(B: &MI == &EI)); |
| 3130 | } |
| 3131 | IM.replaceInstr(OldMI: &MI, NewMI: CopyOut); |
| 3132 | B.erase(I: It); |
| 3133 | } |
| 3134 | |
| 3135 | // Update the dead map. |
| 3136 | HexagonBlockRanges::RegisterRef FoundRR = { .Reg: FoundR, .Sub: 0 }; |
| 3137 | for (auto RR : HexagonBlockRanges::expandToSubRegs(R: FoundRR, MRI, TRI: HRI)) |
| 3138 | DM[RR].subtract(Range); |
| 3139 | } // for Range in range list |
| 3140 | } |
| 3141 | } |
| 3142 | } |
| 3143 | |
| 3144 | void HexagonFrameLowering::expandAlloca(MachineInstr *AI, MachineFunction &MF, |
| 3145 | const HexagonInstrInfo &HII, |
| 3146 | Register SP, unsigned CF) const { |
| 3147 | MachineBasicBlock &MB = *AI->getParent(); |
| 3148 | DebugLoc DL = AI->getDebugLoc(); |
| 3149 | unsigned A = AI->getOperand(i: 2).getImm(); |
| 3150 | |
| 3151 | MachineOperand &RdOp = AI->getOperand(i: 0); |
| 3152 | MachineOperand &RsOp = AI->getOperand(i: 1); |
| 3153 | Register Rd = RdOp.getReg(), Rs = RsOp.getReg(); |
| 3154 | |
| 3155 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 3156 | auto *TLI = HST.getTargetLowering(); |
| 3157 | bool NeedsProbing = TLI->hasInlineStackProbe(MF); |
| 3158 | |
| 3159 | if (!NeedsProbing) { |
| 3160 | // Have |
| 3161 | // Rd = alloca Rs, #A |
| 3162 | // |
| 3163 | // If Rs and Rd are different registers, use this sequence: |
| 3164 | // Rd = sub(r29, Rs) |
| 3165 | // r29 = sub(r29, Rs) |
| 3166 | // Rd = and(Rd, #-A) ; if necessary |
| 3167 | // r29 = and(r29, #-A) ; if necessary |
| 3168 | // Rd = add(Rd, #CF) ; CF size aligned to at most A |
| 3169 | // otherwise, do |
| 3170 | // Rd = sub(r29, Rs) |
| 3171 | // Rd = and(Rd, #-A) ; if necessary |
| 3172 | // r29 = Rd |
| 3173 | // Rd = add(Rd, #CF) ; CF size aligned to at most A |
| 3174 | |
| 3175 | // Rd = sub(r29, Rs) |
| 3176 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_sub), DestReg: Rd).addReg(RegNo: SP).addReg(RegNo: Rs); |
| 3177 | if (Rs != Rd) { |
| 3178 | // r29 = sub(r29, Rs) |
| 3179 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_sub), DestReg: SP).addReg(RegNo: SP).addReg(RegNo: Rs); |
| 3180 | } |
| 3181 | if (A > 8) { |
| 3182 | // Rd = and(Rd, #-A) |
| 3183 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_andir), DestReg: Rd) |
| 3184 | .addReg(RegNo: Rd) |
| 3185 | .addImm(Val: -int64_t(A)); |
| 3186 | if (Rs != Rd) |
| 3187 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_andir), DestReg: SP) |
| 3188 | .addReg(RegNo: SP) |
| 3189 | .addImm(Val: -int64_t(A)); |
| 3190 | } |
| 3191 | if (Rs == Rd) { |
| 3192 | // r29 = Rd |
| 3193 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: TargetOpcode::COPY), DestReg: SP).addReg(RegNo: Rd); |
| 3194 | } |
| 3195 | if (CF > 0) { |
| 3196 | // Rd = add(Rd, #CF) |
| 3197 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: Rd).addReg(RegNo: Rd).addImm(Val: CF); |
| 3198 | } |
| 3199 | return; |
| 3200 | } |
| 3201 | |
| 3202 | // Stack probing for dynamic allocation. The size Rs is a runtime value |
| 3203 | // so the probe loop is always emitted; it is a no-op when Rs is small. |
| 3204 | // |
| 3205 | // Compute the target SP into Rd (with optional alignment), then probe |
| 3206 | // each page on the way down: |
| 3207 | // |
| 3208 | // Rd = sub(r29, Rs) |
| 3209 | // [Rd = and(Rd, #-A)] ; if alignment > 8 |
| 3210 | // LoopMBB: |
| 3211 | // r29 = add(r29, #-ProbeSize) |
| 3212 | // memw(r29+#0) = #0 |
| 3213 | // p0 = cmp.gtu(r29, Rd) |
| 3214 | // if (p0.new) jump:t LoopMBB |
| 3215 | // ExitMBB: |
| 3216 | // r29 = Rd |
| 3217 | // [Rd = add(Rd, #CF)] ; if CF > 0 |
| 3218 | // <rest of original block> |
| 3219 | // |
| 3220 | // Rd holds the exact (aligned) target SP throughout the loop, so the |
| 3221 | // final "r29 = Rd" snaps SP to the correct value even when Rs is not |
| 3222 | // a multiple of ProbeSize. |
| 3223 | |
| 3224 | Align StackAlign = getStackAlign(); |
| 3225 | unsigned ProbeSize = TLI->getStackProbeSize(MF, StackAlign); |
| 3226 | MachineInstr::MIFlag Flags = MachineInstr::FrameSetup; |
| 3227 | |
| 3228 | // Emit target-SP computation into Rd before splitting the block. |
| 3229 | // Rd = sub(r29, Rs) |
| 3230 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_sub), DestReg: Rd) |
| 3231 | .addReg(RegNo: SP) |
| 3232 | .addReg(RegNo: Rs) |
| 3233 | .setMIFlags(Flags); |
| 3234 | if (A > 8) { |
| 3235 | // Rd = and(Rd, #-A) |
| 3236 | BuildMI(BB&: MB, I: AI, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_andir), DestReg: Rd) |
| 3237 | .addReg(RegNo: Rd) |
| 3238 | .addImm(Val: -int64_t(A)) |
| 3239 | .setMIFlags(Flags); |
| 3240 | } |
| 3241 | |
| 3242 | // Split the block: everything after AI goes into ExitMBB. |
| 3243 | MachineFunction::iterator InsertPt = std::next(x: MB.getIterator()); |
| 3244 | MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: MB.getBasicBlock()); |
| 3245 | MF.insert(MBBI: InsertPt, MBB: LoopMBB); |
| 3246 | MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MB.getBasicBlock()); |
| 3247 | MF.insert(MBBI: InsertPt, MBB: ExitMBB); |
| 3248 | |
| 3249 | // Move instructions after AI (exclusive) into ExitMBB. |
| 3250 | ExitMBB->splice(Where: ExitMBB->end(), Other: &MB, From: std::next(x: AI->getIterator()), To: MB.end()); |
| 3251 | ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MB); |
| 3252 | |
| 3253 | // LoopMBB: probe each page. |
| 3254 | // r29 = add(r29, #-ProbeSize) |
| 3255 | // memw(r29+#0) = #0 |
| 3256 | // p0 = cmp.gtu(r29, Rd) |
| 3257 | // if (p0.new) jump:t LoopMBB |
| 3258 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: Hexagon::R29) |
| 3259 | .addReg(RegNo: Hexagon::R29) |
| 3260 | .addImm(Val: -int(ProbeSize)) |
| 3261 | .setMIFlags(Flags); |
| 3262 | |
| 3263 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::S4_storeiri_io)) |
| 3264 | .addReg(RegNo: Hexagon::R29) |
| 3265 | .addImm(Val: 0) |
| 3266 | .addImm(Val: 0) |
| 3267 | .setMIFlags(Flags); |
| 3268 | |
| 3269 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::C2_cmpgtu), |
| 3270 | DestReg: Hexagon::P0) |
| 3271 | .addReg(RegNo: Hexagon::R29) |
| 3272 | .addReg(RegNo: Rd) |
| 3273 | .setMIFlags(Flags); |
| 3274 | |
| 3275 | BuildMI(BB&: *LoopMBB, I: LoopMBB->end(), MIMD: DL, MCID: HII.get(Opcode: Hexagon::J2_jumpt)) |
| 3276 | .addReg(RegNo: Hexagon::P0) |
| 3277 | .addMBB(MBB: LoopMBB) |
| 3278 | .setMIFlags(Flags); |
| 3279 | |
| 3280 | // ExitMBB: snap SP to exact target, then apply CF offset to Rd. |
| 3281 | // r29 = Rd |
| 3282 | // [Rd = add(Rd, #CF)] |
| 3283 | MachineBasicBlock::iterator ExitIt = ExitMBB->begin(); |
| 3284 | BuildMI(BB&: *ExitMBB, I: ExitIt, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_tfr), DestReg: Hexagon::R29) |
| 3285 | .addReg(RegNo: Rd) |
| 3286 | .setMIFlags(Flags); |
| 3287 | if (CF > 0) { |
| 3288 | BuildMI(BB&: *ExitMBB, I: ExitIt, MIMD: DL, MCID: HII.get(Opcode: Hexagon::A2_addi), DestReg: Rd) |
| 3289 | .addReg(RegNo: Rd) |
| 3290 | .addImm(Val: CF) |
| 3291 | .setMIFlags(Flags); |
| 3292 | } |
| 3293 | |
| 3294 | // Wire up CFG edges. |
| 3295 | MB.addSuccessor(Succ: LoopMBB); |
| 3296 | LoopMBB->addSuccessor(Succ: LoopMBB); |
| 3297 | LoopMBB->addSuccessor(Succ: ExitMBB); |
| 3298 | |
| 3299 | // Recompute live-ins for the new blocks. AI is still in MB at this |
| 3300 | // point; the caller erases it after expandAlloca returns. |
| 3301 | fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopMBB}); |
| 3302 | } |
| 3303 | |
| 3304 | bool HexagonFrameLowering::needsAligna(const MachineFunction &MF) const { |
| 3305 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 3306 | if (!MFI.hasVarSizedObjects()) |
| 3307 | return false; |
| 3308 | // Do not check for max stack object alignment here, because the stack |
| 3309 | // may not be complete yet. Assume that we will need PS_aligna if there |
| 3310 | // are variable-sized objects. |
| 3311 | return true; |
| 3312 | } |
| 3313 | |
| 3314 | /// Adds all callee-saved registers as implicit uses or defs to the |
| 3315 | /// instruction. |
| 3316 | void HexagonFrameLowering::addCalleeSaveRegistersAsImpOperand(MachineInstr *MI, |
| 3317 | const CSIVect &CSI, bool IsDef, bool IsKill) const { |
| 3318 | // Add the callee-saved registers as implicit uses. |
| 3319 | for (auto &R : CSI) |
| 3320 | MI->addOperand(Op: MachineOperand::CreateReg(Reg: R.getReg(), isDef: IsDef, isImp: true, isKill: IsKill)); |
| 3321 | } |
| 3322 | |
| 3323 | /// Determine whether the callee-saved register saves and restores should |
| 3324 | /// be generated via inline code. If this function returns "true", inline |
| 3325 | /// code will be generated. If this function returns "false", additional |
| 3326 | /// checks are performed, which may still lead to the inline code. |
| 3327 | bool HexagonFrameLowering::shouldInlineCSR(const MachineFunction &MF, |
| 3328 | const CSIVect &CSI) const { |
| 3329 | if (MF.getSubtarget<HexagonSubtarget>().isEnvironmentMusl()) |
| 3330 | return true; |
| 3331 | if (MF.getInfo<HexagonMachineFunctionInfo>()->hasEHReturn()) |
| 3332 | return true; |
| 3333 | if (!hasFP(MF)) |
| 3334 | return true; |
| 3335 | if (!isOptSize(MF) && !isMinSize(MF)) |
| 3336 | if (MF.getTarget().getOptLevel() > CodeGenOptLevel::Default) |
| 3337 | return true; |
| 3338 | |
| 3339 | // Check if CSI only has double registers, and if the registers form |
| 3340 | // a contiguous block starting from D8. |
| 3341 | BitVector Regs(Hexagon::NUM_TARGET_REGS); |
| 3342 | for (const CalleeSavedInfo &I : CSI) { |
| 3343 | MCRegister R = I.getReg(); |
| 3344 | if (!Hexagon::DoubleRegsRegClass.contains(Reg: R)) |
| 3345 | return true; |
| 3346 | Regs[R] = true; |
| 3347 | } |
| 3348 | int F = Regs.find_first(); |
| 3349 | if (F != Hexagon::D8) |
| 3350 | return true; |
| 3351 | while (F >= 0) { |
| 3352 | int N = Regs.find_next(Prev: F); |
| 3353 | if (N >= 0 && N != F+1) |
| 3354 | return true; |
| 3355 | F = N; |
| 3356 | } |
| 3357 | |
| 3358 | return false; |
| 3359 | } |
| 3360 | |
| 3361 | bool HexagonFrameLowering::useSpillFunction(const MachineFunction &MF, |
| 3362 | const CSIVect &CSI) const { |
| 3363 | if (shouldInlineCSR(MF, CSI)) |
| 3364 | return false; |
| 3365 | unsigned NumCSI = CSI.size(); |
| 3366 | if (NumCSI <= 1) |
| 3367 | return false; |
| 3368 | |
| 3369 | // Every spill stub saves the whole range starting at R16 |
| 3370 | // (__save_r16_through_rNN), so a stub whose range reached the shadow call |
| 3371 | // stack pointer register would spill it along with the real callee-saved |
| 3372 | // registers - and since the SCS register is reserved it is absent from CSI, |
| 3373 | // so the stub's fixed frame layout would not match the one the compiler |
| 3374 | // assigned. |
| 3375 | // |
| 3376 | // shouldInlineCSR() above already makes this unreachable: it only lets a |
| 3377 | // stub through when CSI is a contiguous run of double registers starting at |
| 3378 | // D8, and reserving the SCS register always breaks the double it belongs |
| 3379 | // to, leaving its partner in CSI as a lone single register. This is a |
| 3380 | // cheap safety net so the guarantee does not rest on that reasoning alone. |
| 3381 | if (MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) { |
| 3382 | const auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 3383 | Register MaxReg = getMaxCalleeSavedReg(CSI, TRI: *HST.getRegisterInfo()); |
| 3384 | if (HST.getSCSPReg().id() <= MaxReg.id()) |
| 3385 | return false; |
| 3386 | } |
| 3387 | |
| 3388 | unsigned Threshold = isOptSize(MF) ? SpillFuncThresholdOs |
| 3389 | : SpillFuncThreshold; |
| 3390 | return Threshold < NumCSI; |
| 3391 | } |
| 3392 | |
| 3393 | bool HexagonFrameLowering::useRestoreFunction(const MachineFunction &MF, |
| 3394 | const CSIVect &CSI) const { |
| 3395 | if (shouldInlineCSR(MF, CSI)) |
| 3396 | return false; |
| 3397 | // The returning restore stubs do jumpr r31, this breaks ShadowCallStack: |
| 3398 | if (MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack)) |
| 3399 | return false; |
| 3400 | // The restore functions do a bit more than just restoring registers. |
| 3401 | // The non-returning versions will go back directly to the caller's |
| 3402 | // caller, others will clean up the stack frame in preparation for |
| 3403 | // a tail call. Using them can still save code size even if only one |
| 3404 | // register is getting restores. Make the decision based on -Oz: |
| 3405 | // using -Os will use inline restore for a single register. |
| 3406 | if (isMinSize(MF)) |
| 3407 | return true; |
| 3408 | unsigned NumCSI = CSI.size(); |
| 3409 | if (NumCSI <= 1) |
| 3410 | return false; |
| 3411 | |
| 3412 | unsigned Threshold = isOptSize(MF) ? SpillFuncThresholdOs-1 |
| 3413 | : SpillFuncThreshold; |
| 3414 | return Threshold < NumCSI; |
| 3415 | } |
| 3416 | |
| 3417 | bool HexagonFrameLowering::mayOverflowFrameOffset(MachineFunction &MF) const { |
| 3418 | unsigned StackSize = MF.getFrameInfo().estimateStackSize(MF); |
| 3419 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 3420 | // A fairly simplistic guess as to whether a potential load/store to a |
| 3421 | // stack location could require an extra register. |
| 3422 | if (HST.useHVXOps() && StackSize > 256) |
| 3423 | return true; |
| 3424 | |
| 3425 | // Check if the function has store-immediate instructions that access |
| 3426 | // the stack. Since the offset field is not extendable, if the stack |
| 3427 | // size exceeds the offset limit (6 bits, shifted), the stores will |
| 3428 | // require a new base register. |
| 3429 | bool HasImmStack = false; |
| 3430 | unsigned MinLS = ~0u; // Log_2 of the memory access size. |
| 3431 | |
| 3432 | for (const MachineBasicBlock &B : MF) { |
| 3433 | for (const MachineInstr &MI : B) { |
| 3434 | unsigned LS = 0; |
| 3435 | switch (MI.getOpcode()) { |
| 3436 | case Hexagon::S4_storeirit_io: |
| 3437 | case Hexagon::S4_storeirif_io: |
| 3438 | case Hexagon::S4_storeiri_io: |
| 3439 | ++LS; |
| 3440 | [[fallthrough]]; |
| 3441 | case Hexagon::S4_storeirht_io: |
| 3442 | case Hexagon::S4_storeirhf_io: |
| 3443 | case Hexagon::S4_storeirh_io: |
| 3444 | ++LS; |
| 3445 | [[fallthrough]]; |
| 3446 | case Hexagon::S4_storeirbt_io: |
| 3447 | case Hexagon::S4_storeirbf_io: |
| 3448 | case Hexagon::S4_storeirb_io: |
| 3449 | if (MI.getOperand(i: 0).isFI()) |
| 3450 | HasImmStack = true; |
| 3451 | MinLS = std::min(a: MinLS, b: LS); |
| 3452 | break; |
| 3453 | } |
| 3454 | } |
| 3455 | } |
| 3456 | |
| 3457 | if (HasImmStack) |
| 3458 | return !isUInt<6>(x: StackSize >> MinLS); |
| 3459 | |
| 3460 | return false; |
| 3461 | } |
| 3462 | |
| 3463 | namespace { |
| 3464 | // Struct used by orderFrameObjects to help sort the stack objects. |
| 3465 | struct HexagonFrameSortingObject { |
| 3466 | bool IsValid = false; |
| 3467 | unsigned Index = 0; // Index of Object into MFI list. |
| 3468 | unsigned Size = 0; |
| 3469 | Align ObjectAlignment = Align(1); // Alignment of Object in bytes. |
| 3470 | }; |
| 3471 | |
| 3472 | struct HexagonFrameSortingComparator { |
| 3473 | inline bool operator()(const HexagonFrameSortingObject &A, |
| 3474 | const HexagonFrameSortingObject &B) const { |
| 3475 | return std::make_tuple(args: !A.IsValid, args: A.ObjectAlignment, args: A.Size) < |
| 3476 | std::make_tuple(args: !B.IsValid, args: B.ObjectAlignment, args: B.Size); |
| 3477 | } |
| 3478 | }; |
| 3479 | } // namespace |
| 3480 | |
| 3481 | // Sort objects on the stack by alignment value and then by size to minimize |
| 3482 | // padding. |
| 3483 | void HexagonFrameLowering::orderFrameObjects( |
| 3484 | const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const { |
| 3485 | |
| 3486 | if (ObjectsToAllocate.empty()) |
| 3487 | return; |
| 3488 | |
| 3489 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 3490 | int NObjects = ObjectsToAllocate.size(); |
| 3491 | |
| 3492 | // Create an array of all MFI objects. |
| 3493 | SmallVector<HexagonFrameSortingObject> SortingObjects( |
| 3494 | MFI.getObjectIndexEnd()); |
| 3495 | |
| 3496 | for (int i = 0, j = 0, e = MFI.getObjectIndexEnd(); i < e && j != NObjects; |
| 3497 | ++i) { |
| 3498 | if (i != ObjectsToAllocate[j]) |
| 3499 | continue; |
| 3500 | j++; |
| 3501 | |
| 3502 | // A variable size object has size equal to 0. Since Hexagon sets |
| 3503 | // getUseLocalStackAllocationBlock() to true, a local block is allocated |
| 3504 | // earlier. This case is not handled here for now. |
| 3505 | int Size = MFI.getObjectSize(ObjectIdx: i); |
| 3506 | if (Size == 0) |
| 3507 | return; |
| 3508 | |
| 3509 | SortingObjects[i].IsValid = true; |
| 3510 | SortingObjects[i].Index = i; |
| 3511 | SortingObjects[i].Size = Size; |
| 3512 | SortingObjects[i].ObjectAlignment = MFI.getObjectAlign(ObjectIdx: i); |
| 3513 | } |
| 3514 | |
| 3515 | // Sort objects by alignment and then by size. |
| 3516 | llvm::stable_sort(Range&: SortingObjects, C: HexagonFrameSortingComparator()); |
| 3517 | |
| 3518 | // Modify the original list to represent the final order. |
| 3519 | int i = NObjects; |
| 3520 | for (auto &Obj : SortingObjects) { |
| 3521 | if (i == 0) |
| 3522 | break; |
| 3523 | ObjectsToAllocate[--i] = Obj.Index; |
| 3524 | } |
| 3525 | } |
| 3526 | |