1//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetInstrInfo.h"
14#include "llvm/ADT/SmallSet.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/Dwarf.h"
17#include "llvm/CodeGen/MachineCombinerPattern.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineMemOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/MachineScheduler.h"
23#include "llvm/CodeGen/MachineTraceMetrics.h"
24#include "llvm/CodeGen/PseudoSourceValue.h"
25#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
26#include "llvm/CodeGen/StackMaps.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/CodeGen/TargetLowering.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/CodeGen/TargetSchedule.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/MC/MCAsmInfo.h"
34#include "llvm/MC/MCInstrItineraries.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/InterleavedRange.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Target/TargetMachine.h"
40
41using namespace llvm;
42
43static cl::opt<bool> DisableHazardRecognizer(
44 "disable-sched-hazard", cl::Hidden, cl::init(Val: false),
45 cl::desc("Disable hazard detection during preRA scheduling"));
46
47static cl::opt<bool> EnableAccReassociation(
48 "acc-reassoc", cl::Hidden, cl::init(Val: true),
49 cl::desc("Enable reassociation of accumulation chains"));
50
51static cl::opt<unsigned int>
52 MinAccumulatorDepth("acc-min-depth", cl::Hidden, cl::init(Val: 8),
53 cl::desc("Minimum length of accumulator chains "
54 "required for the optimization to kick in"));
55
56static cl::opt<unsigned int> MaxAccumulatorWidth(
57 "acc-max-width", cl::Hidden, cl::init(Val: 3),
58 cl::desc("Maximum number of branches in the accumulator tree"));
59
60TargetInstrInfo::~TargetInstrInfo() = default;
61
62const TargetRegisterClass *TargetInstrInfo::getRegClass(const MCInstrDesc &MCID,
63 unsigned OpNum) const {
64 if (OpNum >= MCID.getNumOperands())
65 return nullptr;
66
67 const MCOperandInfo &OpInfo = MCID.operands()[OpNum];
68 int16_t RegClass = getOpRegClassID(OpInfo);
69
70 // Instructions like INSERT_SUBREG do not have fixed register classes.
71 if (RegClass < 0)
72 return nullptr;
73
74 // Otherwise just look it up normally.
75 return TRI.getRegClass(i: RegClass);
76}
77
78/// insertNoop - Insert a noop into the instruction stream at the specified
79/// point.
80void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB,
81 MachineBasicBlock::iterator MI) const {
82 llvm_unreachable("Target didn't implement insertNoop!");
83}
84
85/// insertNoops - Insert noops into the instruction stream at the specified
86/// point.
87void TargetInstrInfo::insertNoops(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator MI,
89 unsigned Quantity) const {
90 for (unsigned i = 0; i < Quantity; ++i)
91 insertNoop(MBB, MI);
92}
93
94static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
95 return strncmp(s1: Str, s2: MAI.getCommentString().data(),
96 n: MAI.getCommentString().size()) == 0;
97}
98
99/// Measure the specified inline asm to determine an approximation of its
100/// length.
101/// Comments (which run till the next SeparatorString or newline) do not
102/// count as an instruction.
103/// Any other non-whitespace text is considered an instruction, with
104/// multiple instructions separated by SeparatorString or newlines.
105/// Variable-length instructions are not handled here; this function
106/// may be overloaded in the target code to do that.
107/// We implement a special case of the .space directive which takes only a
108/// single integer argument in base 10 that is the size in bytes. This is a
109/// restricted form of the GAS directive in that we only interpret
110/// simple--i.e. not a logical or arithmetic expression--size values without
111/// the optional fill value. This is primarily used for creating arbitrary
112/// sized inline asm blocks for testing purposes.
113unsigned TargetInstrInfo::getInlineAsmLength(
114 const char *Str,
115 const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const {
116 // Count the number of instructions in the asm.
117 bool AtInsnStart = true;
118 unsigned Length = 0;
119 const unsigned MaxInstLength = MAI.getMaxInstLength(STI);
120 for (; *Str; ++Str) {
121 if (*Str == '\n' || strncmp(s1: Str, s2: MAI.getSeparatorString(),
122 n: strlen(s: MAI.getSeparatorString())) == 0) {
123 AtInsnStart = true;
124 } else if (isAsmComment(Str, MAI)) {
125 // Stop counting as an instruction after a comment until the next
126 // separator.
127 AtInsnStart = false;
128 }
129
130 if (AtInsnStart && !isSpace(C: static_cast<unsigned char>(*Str))) {
131 unsigned AddLength = MaxInstLength;
132 if (strncmp(s1: Str, s2: ".space", n: 6) == 0) {
133 char *EStr;
134 int SpaceSize;
135 SpaceSize = strtol(nptr: Str + 6, endptr: &EStr, base: 10);
136 SpaceSize = SpaceSize < 0 ? 0 : SpaceSize;
137 while (*EStr != '\n' && isSpace(C: static_cast<unsigned char>(*EStr)))
138 ++EStr;
139 if (*EStr == '\0' || *EStr == '\n' ||
140 isAsmComment(Str: EStr, MAI)) // Successfully parsed .space argument
141 AddLength = SpaceSize;
142 }
143 Length += AddLength;
144 AtInsnStart = false;
145 }
146 }
147
148 return Length;
149}
150
151/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
152/// after it, replacing it with an unconditional branch to NewDest.
153void
154TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
155 MachineBasicBlock *NewDest) const {
156 MachineBasicBlock *MBB = Tail->getParent();
157
158 // Remove all the old successors of MBB from the CFG.
159 while (!MBB->succ_empty())
160 MBB->removeSuccessor(I: MBB->succ_begin());
161
162 // Save off the debug loc before erasing the instruction.
163 DebugLoc DL = Tail->getDebugLoc();
164
165 // Update call info and remove all the dead instructions
166 // from the end of MBB.
167 while (Tail != MBB->end()) {
168 auto MI = Tail++;
169 if (MI->shouldUpdateAdditionalCallInfo())
170 MBB->getParent()->eraseAdditionalCallInfo(MI: &*MI);
171 MBB->erase(I: MI);
172 }
173
174 // If MBB isn't immediately before MBB, insert a branch to it.
175 if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
176 insertBranch(MBB&: *MBB, TBB: NewDest, FBB: nullptr, Cond: SmallVector<MachineOperand, 0>(), DL);
177 MBB->addSuccessor(Succ: NewDest);
178}
179
180MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI,
181 bool NewMI, unsigned Idx1,
182 unsigned Idx2) const {
183 const MCInstrDesc &MCID = MI.getDesc();
184 bool HasDef = MCID.getNumDefs();
185 if (HasDef && !MI.getOperand(i: 0).isReg())
186 // No idea how to commute this instruction. Target should implement its own.
187 return nullptr;
188
189 unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
190 unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
191 assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
192 CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
193 "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
194 assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
195 "This only knows how to commute register operands so far");
196
197 Register Reg0 = HasDef ? MI.getOperand(i: 0).getReg() : Register();
198 Register Reg1 = MI.getOperand(i: Idx1).getReg();
199 Register Reg2 = MI.getOperand(i: Idx2).getReg();
200 unsigned SubReg0 = HasDef ? MI.getOperand(i: 0).getSubReg() : 0;
201 unsigned SubReg1 = MI.getOperand(i: Idx1).getSubReg();
202 unsigned SubReg2 = MI.getOperand(i: Idx2).getSubReg();
203 bool Reg1IsKill = MI.getOperand(i: Idx1).isKill();
204 bool Reg2IsKill = MI.getOperand(i: Idx2).isKill();
205 bool Reg1IsUndef = MI.getOperand(i: Idx1).isUndef();
206 bool Reg2IsUndef = MI.getOperand(i: Idx2).isUndef();
207 bool Reg1IsInternal = MI.getOperand(i: Idx1).isInternalRead();
208 bool Reg2IsInternal = MI.getOperand(i: Idx2).isInternalRead();
209 // Avoid calling isRenamable for virtual registers since we assert that
210 // renamable property is only queried/set for physical registers.
211 bool Reg1IsRenamable =
212 Reg1.isPhysical() ? MI.getOperand(i: Idx1).isRenamable() : false;
213 bool Reg2IsRenamable =
214 Reg2.isPhysical() ? MI.getOperand(i: Idx2).isRenamable() : false;
215
216 // For a case like this:
217 // %0.sub = INST %0.sub(tied), %1.sub, implicit-def %0
218 // we need to update the implicit-def after commuting to result in:
219 // %1.sub = INST %1.sub(tied), %0.sub, implicit-def %1
220 SmallVector<unsigned> UpdateImplicitDefIdx;
221 if (HasDef && MI.hasImplicitDef()) {
222 for (auto [OpNo, MO] : llvm::enumerate(First: MI.implicit_operands())) {
223 Register ImplReg = MO.getReg();
224 if ((ImplReg.isVirtual() && ImplReg == Reg0) ||
225 (ImplReg.isPhysical() && Reg0.isPhysical() &&
226 TRI.isSubRegisterEq(RegA: ImplReg, RegB: Reg0)))
227 UpdateImplicitDefIdx.push_back(Elt: OpNo + MI.getNumExplicitOperands());
228 }
229 }
230
231 // If destination is tied to either of the commuted source register, then
232 // it must be updated.
233 if (HasDef && Reg0 == Reg1 &&
234 MI.getDesc().getOperandConstraint(OpNum: Idx1, Constraint: MCOI::TIED_TO) == 0) {
235 Reg2IsKill = false;
236 Reg0 = Reg2;
237 SubReg0 = SubReg2;
238 } else if (HasDef && Reg0 == Reg2 &&
239 MI.getDesc().getOperandConstraint(OpNum: Idx2, Constraint: MCOI::TIED_TO) == 0) {
240 Reg1IsKill = false;
241 Reg0 = Reg1;
242 SubReg0 = SubReg1;
243 }
244
245 MachineInstr *CommutedMI = nullptr;
246 if (NewMI) {
247 // Create a new instruction.
248 MachineFunction &MF = *MI.getMF();
249 CommutedMI = MF.CloneMachineInstr(Orig: &MI);
250 } else {
251 CommutedMI = &MI;
252 }
253
254 if (HasDef) {
255 CommutedMI->getOperand(i: 0).setReg(Reg0);
256 CommutedMI->getOperand(i: 0).setSubReg(SubReg0);
257 for (unsigned Idx : UpdateImplicitDefIdx)
258 CommutedMI->getOperand(i: Idx).setReg(Reg0);
259 }
260 CommutedMI->getOperand(i: Idx2).setReg(Reg1);
261 CommutedMI->getOperand(i: Idx1).setReg(Reg2);
262 CommutedMI->getOperand(i: Idx2).setSubReg(SubReg1);
263 CommutedMI->getOperand(i: Idx1).setSubReg(SubReg2);
264 CommutedMI->getOperand(i: Idx2).setIsKill(Reg1IsKill);
265 CommutedMI->getOperand(i: Idx1).setIsKill(Reg2IsKill);
266 CommutedMI->getOperand(i: Idx2).setIsUndef(Reg1IsUndef);
267 CommutedMI->getOperand(i: Idx1).setIsUndef(Reg2IsUndef);
268 CommutedMI->getOperand(i: Idx2).setIsInternalRead(Reg1IsInternal);
269 CommutedMI->getOperand(i: Idx1).setIsInternalRead(Reg2IsInternal);
270 // Avoid calling setIsRenamable for virtual registers since we assert that
271 // renamable property is only queried/set for physical registers.
272 if (Reg1.isPhysical())
273 CommutedMI->getOperand(i: Idx2).setIsRenamable(Reg1IsRenamable);
274 if (Reg2.isPhysical())
275 CommutedMI->getOperand(i: Idx1).setIsRenamable(Reg2IsRenamable);
276 return CommutedMI;
277}
278
279MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI,
280 unsigned OpIdx1,
281 unsigned OpIdx2) const {
282 // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
283 // any commutable operand, which is done in findCommutedOpIndices() method
284 // called below.
285 if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) &&
286 !findCommutedOpIndices(MI, SrcOpIdx1&: OpIdx1, SrcOpIdx2&: OpIdx2)) {
287 assert(MI.isCommutable() &&
288 "Precondition violation: MI must be commutable.");
289 return nullptr;
290 }
291 return commuteInstructionImpl(MI, NewMI, Idx1: OpIdx1, Idx2: OpIdx2);
292}
293
294bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
295 unsigned &ResultIdx2,
296 unsigned CommutableOpIdx1,
297 unsigned CommutableOpIdx2) {
298 if (ResultIdx1 == CommuteAnyOperandIndex &&
299 ResultIdx2 == CommuteAnyOperandIndex) {
300 ResultIdx1 = CommutableOpIdx1;
301 ResultIdx2 = CommutableOpIdx2;
302 } else if (ResultIdx1 == CommuteAnyOperandIndex) {
303 if (ResultIdx2 == CommutableOpIdx1)
304 ResultIdx1 = CommutableOpIdx2;
305 else if (ResultIdx2 == CommutableOpIdx2)
306 ResultIdx1 = CommutableOpIdx1;
307 else
308 return false;
309 } else if (ResultIdx2 == CommuteAnyOperandIndex) {
310 if (ResultIdx1 == CommutableOpIdx1)
311 ResultIdx2 = CommutableOpIdx2;
312 else if (ResultIdx1 == CommutableOpIdx2)
313 ResultIdx2 = CommutableOpIdx1;
314 else
315 return false;
316 } else
317 // Check that the result operand indices match the given commutable
318 // operand indices.
319 return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) ||
320 (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1);
321
322 return true;
323}
324
325bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
326 unsigned &SrcOpIdx1,
327 unsigned &SrcOpIdx2) const {
328 assert(!MI.isBundle() &&
329 "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
330
331 const MCInstrDesc &MCID = MI.getDesc();
332 if (!MCID.isCommutable())
333 return false;
334
335 // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
336 // is not true, then the target must implement this.
337 unsigned CommutableOpIdx1 = MCID.getNumDefs();
338 unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
339 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2,
340 CommutableOpIdx1, CommutableOpIdx2))
341 return false;
342
343 if (!MI.getOperand(i: SrcOpIdx1).isReg() || !MI.getOperand(i: SrcOpIdx2).isReg())
344 // No idea.
345 return false;
346 return true;
347}
348
349bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
350 if (!MI.isTerminator()) return false;
351
352 // Conditional branch is a special case.
353 if (MI.isBranch() && !MI.isBarrier())
354 return true;
355 if (!MI.isPredicable())
356 return true;
357 return !isPredicated(MI);
358}
359
360bool TargetInstrInfo::PredicateInstruction(
361 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
362 bool MadeChange = false;
363
364 assert(!MI.isBundle() &&
365 "TargetInstrInfo::PredicateInstruction() can't handle bundles");
366
367 const MCInstrDesc &MCID = MI.getDesc();
368 if (!MI.isPredicable())
369 return false;
370
371 for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
372 if (MCID.operands()[i].isPredicate()) {
373 MachineOperand &MO = MI.getOperand(i);
374 if (MO.isReg()) {
375 MO.setReg(Pred[j].getReg());
376 MadeChange = true;
377 } else if (MO.isImm()) {
378 MO.setImm(Pred[j].getImm());
379 MadeChange = true;
380 } else if (MO.isMBB()) {
381 MO.setMBB(Pred[j].getMBB());
382 MadeChange = true;
383 }
384 ++j;
385 }
386 }
387 return MadeChange;
388}
389
390bool TargetInstrInfo::hasLoadFromStackSlot(
391 const MachineInstr &MI,
392 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
393 size_t StartSize = Accesses.size();
394 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
395 oe = MI.memoperands_end();
396 o != oe; ++o) {
397 if ((*o)->isLoad() &&
398 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
399 Accesses.push_back(Elt: *o);
400 }
401 return Accesses.size() != StartSize;
402}
403
404bool TargetInstrInfo::hasStoreToStackSlot(
405 const MachineInstr &MI,
406 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
407 size_t StartSize = Accesses.size();
408 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
409 oe = MI.memoperands_end();
410 o != oe; ++o) {
411 if ((*o)->isStore() &&
412 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
413 Accesses.push_back(Elt: *o);
414 }
415 return Accesses.size() != StartSize;
416}
417
418bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
419 unsigned SubIdx, unsigned &Size,
420 unsigned &Offset,
421 const MachineFunction &MF) const {
422 if (!SubIdx) {
423 Size = TRI.getSpillSize(RC: *RC);
424 Offset = 0;
425 return true;
426 }
427 unsigned BitSize = TRI.getSubRegIdxSize(Idx: SubIdx);
428 // Convert bit size to byte size.
429 if (BitSize % 8)
430 return false;
431
432 int BitOffset = TRI.getSubRegIdxOffset(Idx: SubIdx);
433 if (BitOffset < 0 || BitOffset % 8)
434 return false;
435
436 Size = BitSize / 8;
437 Offset = (unsigned)BitOffset / 8;
438
439 assert(TRI.getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
440
441 if (!MF.getDataLayout().isLittleEndian()) {
442 Offset = TRI.getSpillSize(RC: *RC) - (Offset + Size);
443 }
444 return true;
445}
446
447void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
448 MachineBasicBlock::iterator I,
449 Register DestReg, unsigned SubIdx,
450 const MachineInstr &Orig,
451 LaneBitmask UsedLanes) const {
452 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
453 MI->substituteRegister(FromReg: MI->getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
454 MBB.insert(I, MI);
455}
456
457bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0,
458 const MachineInstr &MI1,
459 const MachineRegisterInfo *MRI) const {
460 return MI0.isIdenticalTo(Other: MI1, Check: MachineInstr::IgnoreVRegDefs);
461}
462
463MachineInstr &
464TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
465 MachineBasicBlock::iterator InsertBefore,
466 const MachineInstr &Orig) const {
467 MachineFunction &MF = *MBB.getParent();
468 // CFI instructions are marked as non-duplicable, because Darwin compact
469 // unwind info emission can't handle multiple prologue setups.
470 assert((!Orig.isNotDuplicable() ||
471 (!MF.getTarget().getTargetTriple().isOSDarwin() &&
472 Orig.isCFIInstruction())) &&
473 "Instruction cannot be duplicated");
474
475 return MF.cloneMachineInstrBundle(MBB, InsertBefore, Orig);
476}
477
478// If the COPY instruction in MI can be folded to a stack operation, return
479// the register class to use.
480static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI,
481 const TargetInstrInfo &TII,
482 unsigned FoldIdx) {
483 assert(TII.isCopyInstr(MI) && "MI must be a COPY instruction");
484 if (MI.getNumOperands() != 2)
485 return nullptr;
486 assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
487
488 const MachineOperand &FoldOp = MI.getOperand(i: FoldIdx);
489 const MachineOperand &LiveOp = MI.getOperand(i: 1 - FoldIdx);
490
491 if (FoldOp.getSubReg() || LiveOp.getSubReg())
492 return nullptr;
493
494 Register FoldReg = FoldOp.getReg();
495 Register LiveReg = LiveOp.getReg();
496
497 assert(FoldReg.isVirtual() && "Cannot fold physregs");
498
499 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
500 const TargetRegisterClass *RC = MRI.getRegClass(Reg: FoldReg);
501
502 if (LiveOp.getReg().isPhysical())
503 return RC->contains(Reg: LiveOp.getReg()) ? RC : nullptr;
504
505 if (RC->hasSubClassEq(RC: MRI.getRegClass(Reg: LiveReg)))
506 return RC;
507
508 // FIXME: Allow folding when register classes are memory compatible.
509 return nullptr;
510}
511
512MCInst TargetInstrInfo::getNop() const { llvm_unreachable("Not implemented"); }
513
514/// Try to remove the load by folding it to a register
515/// operand at the use. We fold the load instructions if load defines a virtual
516/// register, the virtual register is used once in the same BB, and the
517/// instructions in-between do not load or store, and have no side effects.
518MachineInstr *TargetInstrInfo::optimizeLoadInstr(MachineInstr &MI,
519 const MachineRegisterInfo *MRI,
520 Register &FoldAsLoadDefReg,
521 MachineInstr *&DefMI) const {
522 // Check whether we can move DefMI here.
523 DefMI = MRI->getVRegDef(Reg: FoldAsLoadDefReg);
524 assert(DefMI);
525 bool SawStore = false;
526 if (!DefMI->isSafeToMove(SawStore))
527 return nullptr;
528
529 // Collect information about virtual register operands of MI.
530 SmallVector<unsigned, 1> SrcOperandIds;
531 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
532 MachineOperand &MO = MI.getOperand(i);
533 if (!MO.isReg())
534 continue;
535 Register Reg = MO.getReg();
536 if (Reg != FoldAsLoadDefReg)
537 continue;
538 // Do not fold if we have a subreg use or a def.
539 if (MO.getSubReg() || MO.isDef())
540 return nullptr;
541 SrcOperandIds.push_back(Elt: i);
542 }
543 if (SrcOperandIds.empty())
544 return nullptr;
545
546 // Check whether we can fold the def into SrcOperandId.
547 if (MachineInstr *FoldMI = foldMemoryOperand(MI, Ops: SrcOperandIds, LoadMI&: *DefMI)) {
548 FoldAsLoadDefReg = 0;
549 return FoldMI;
550 }
551
552 return nullptr;
553}
554
555std::pair<unsigned, unsigned>
556TargetInstrInfo::getPatchpointUnfoldableRange(const MachineInstr &MI) const {
557 switch (MI.getOpcode()) {
558 case TargetOpcode::STACKMAP:
559 // StackMapLiveValues are foldable
560 return std::make_pair(x: 0, y: StackMapOpers(&MI).getVarIdx());
561 case TargetOpcode::PATCHPOINT:
562 // For PatchPoint, the call args are not foldable (even if reported in the
563 // stackmap e.g. via anyregcc).
564 return std::make_pair(x: 0, y: PatchPointOpers(&MI).getVarIdx());
565 case TargetOpcode::STATEPOINT:
566 // For statepoints, fold deopt and gc arguments, but not call arguments.
567 return std::make_pair(x: MI.getNumDefs(), y: StatepointOpers(&MI).getVarIdx());
568 default:
569 llvm_unreachable("unexpected stackmap opcode");
570 }
571}
572
573static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
574 ArrayRef<unsigned> Ops, int FrameIndex,
575 const TargetInstrInfo &TII) {
576 unsigned StartIdx = 0;
577 unsigned NumDefs = 0;
578 // getPatchpointUnfoldableRange throws guarantee if MI is not a patchpoint.
579 std::tie(args&: NumDefs, args&: StartIdx) = TII.getPatchpointUnfoldableRange(MI);
580
581 unsigned DefToFoldIdx = MI.getNumOperands();
582
583 // Return false if any operands requested for folding are not foldable (not
584 // part of the stackmap's live values).
585 for (unsigned Op : Ops) {
586 if (Op < NumDefs) {
587 assert(DefToFoldIdx == MI.getNumOperands() && "Folding multiple defs");
588 DefToFoldIdx = Op;
589 } else if (Op < StartIdx) {
590 return nullptr;
591 }
592 if (MI.getOperand(i: Op).isTied())
593 return nullptr;
594 }
595
596 MachineInstr *NewMI =
597 MF.CreateMachineInstr(MCID: TII.get(Opcode: MI.getOpcode()), DL: MI.getDebugLoc(), NoImplicit: true);
598 MachineInstrBuilder MIB(MF, NewMI);
599
600 // No need to fold return, the meta data, and function arguments
601 for (unsigned i = 0; i < StartIdx; ++i)
602 if (i != DefToFoldIdx)
603 MIB.add(MO: MI.getOperand(i));
604
605 for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) {
606 MachineOperand &MO = MI.getOperand(i);
607 unsigned TiedTo = e;
608 (void)MI.isRegTiedToDefOperand(UseOpIdx: i, DefOpIdx: &TiedTo);
609
610 if (is_contained(Range&: Ops, Element: i)) {
611 assert(TiedTo == e && "Cannot fold tied operands");
612 unsigned SpillSize;
613 unsigned SpillOffset;
614 // Compute the spill slot size and offset.
615 const TargetRegisterClass *RC =
616 MF.getRegInfo().getRegClass(Reg: MO.getReg());
617 bool Valid =
618 TII.getStackSlotRange(RC, SubIdx: MO.getSubReg(), Size&: SpillSize, Offset&: SpillOffset, MF);
619 if (!Valid)
620 report_fatal_error(reason: "cannot spill patchpoint subregister operand");
621 MIB.addImm(Val: StackMaps::IndirectMemRefOp);
622 MIB.addImm(Val: SpillSize);
623 MIB.addFrameIndex(Idx: FrameIndex);
624 MIB.addImm(Val: SpillOffset);
625 } else {
626 MIB.add(MO);
627 if (TiedTo < e) {
628 assert(TiedTo < NumDefs && "Bad tied operand");
629 if (TiedTo > DefToFoldIdx)
630 --TiedTo;
631 NewMI->tieOperands(DefIdx: TiedTo, UseIdx: NewMI->getNumOperands() - 1);
632 }
633 }
634 }
635 return NewMI;
636}
637
638static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI,
639 const TargetInstrInfo &TII) {
640 // If the machine operand is tied, untie it first.
641 if (MI->getOperand(i: OpNo).isTied()) {
642 unsigned TiedTo = MI->findTiedOperandIdx(OpIdx: OpNo);
643 MI->untieRegOperand(OpIdx: OpNo);
644 // Intentional recursion!
645 foldInlineAsmMemOperand(MI, OpNo: TiedTo, FI, TII);
646 }
647
648 SmallVector<MachineOperand, 5> NewOps;
649 TII.getFrameIndexOperands(Ops&: NewOps, FI);
650 assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands");
651 MI->removeOperand(OpNo);
652 MI->insert(InsertBefore: MI->operands_begin() + OpNo, Ops: NewOps);
653
654 // Change the previous operand to a MemKind InlineAsm::Flag. The second param
655 // is the per-target number of operands that represent the memory operand
656 // excluding this one (MD). This includes MO.
657 InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size());
658 F.setMemConstraint(InlineAsm::ConstraintCode::m);
659 MachineOperand &MD = MI->getOperand(i: OpNo - 1);
660 MD.setImm(F);
661}
662
663// Returns nullptr if not possible to fold.
664static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
665 ArrayRef<unsigned> Ops, int FI,
666 const TargetInstrInfo &TII) {
667 assert(MI.isInlineAsm() && "wrong opcode");
668 if (Ops.size() > 1)
669 return nullptr;
670 unsigned Op = Ops[0];
671 assert(Op && "should never be first operand");
672 assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands");
673
674 if (!MI.mayFoldInlineAsmRegOp(OpId: Op))
675 return nullptr;
676
677 MachineInstr &NewMI = TII.duplicate(MBB&: *MI.getParent(), InsertBefore: MI.getIterator(), Orig: MI);
678
679 foldInlineAsmMemOperand(MI: &NewMI, OpNo: Op, FI, TII);
680
681 // Update mayload/maystore metadata, and memoperands.
682 const VirtRegInfo &RI =
683 AnalyzeVirtRegInBundle(MI, Reg: MI.getOperand(i: Op).getReg());
684 MachineOperand &ExtraMO = NewMI.getOperand(i: InlineAsm::MIOp_ExtraInfo);
685 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
686 if (RI.Reads) {
687 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad);
688 Flags |= MachineMemOperand::MOLoad;
689 }
690 if (RI.Writes) {
691 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
692 Flags |= MachineMemOperand::MOStore;
693 }
694 MachineFunction *MF = NewMI.getMF();
695 const MachineFrameInfo &MFI = MF->getFrameInfo();
696 MachineMemOperand *MMO = MF->getMachineMemOperand(
697 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: Flags, Size: MFI.getObjectSize(ObjectIdx: FI),
698 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
699 NewMI.addMemOperand(MF&: *MF, MO: MMO);
700
701 return &NewMI;
702}
703
704MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
705 ArrayRef<unsigned> Ops, int FI,
706 LiveIntervals *LIS,
707 VirtRegMap *VRM) const {
708 auto Flags = MachineMemOperand::MONone;
709 for (unsigned OpIdx : Ops)
710 Flags |= MI.getOperand(i: OpIdx).isDef() ? MachineMemOperand::MOStore
711 : MachineMemOperand::MOLoad;
712
713 MachineBasicBlock *MBB = MI.getParent();
714 assert(MBB && "foldMemoryOperand needs an inserted instruction");
715 MachineFunction &MF = *MBB->getParent();
716
717 // If we're not folding a load into a subreg, the size of the load is the
718 // size of the spill slot. But if we are, we need to figure out what the
719 // actual load size is.
720 int64_t MemSize = 0;
721 const MachineFrameInfo &MFI = MF.getFrameInfo();
722
723 if (Flags & MachineMemOperand::MOStore) {
724 MemSize = MFI.getObjectSize(ObjectIdx: FI);
725 } else {
726 for (unsigned OpIdx : Ops) {
727 int64_t OpSize = MFI.getObjectSize(ObjectIdx: FI);
728
729 if (auto SubReg = MI.getOperand(i: OpIdx).getSubReg()) {
730 unsigned SubRegSize = TRI.getSubRegIdxSize(Idx: SubReg);
731 if (SubRegSize > 0 && !(SubRegSize % 8))
732 OpSize = SubRegSize / 8;
733 }
734
735 MemSize = std::max(a: MemSize, b: OpSize);
736 }
737 }
738
739 assert(MemSize && "Did not expect a zero-sized stack slot");
740
741 MachineInstr *NewMI = nullptr;
742
743 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
744 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
745 MI.getOpcode() == TargetOpcode::STATEPOINT) {
746 // Fold stackmap/patchpoint.
747 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex: FI, TII: *this);
748 if (NewMI)
749 MBB->insert(I: MI, MI: NewMI);
750 } else if (MI.isInlineAsm()) {
751 return foldInlineAsmMemOperand(MI, Ops, FI, TII: *this);
752 } else {
753 // Ask the target to do the actual folding.
754 NewMI = foldMemoryOperandImpl(MF, MI, Ops, InsertPt: MI, FrameIndex: FI, LIS, VRM);
755 }
756
757 if (NewMI) {
758 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
759 // Add a memory operand, foldMemoryOperandImpl doesn't do that.
760 assert((!(Flags & MachineMemOperand::MOStore) ||
761 NewMI->mayStore()) &&
762 "Folded a def to a non-store!");
763 assert((!(Flags & MachineMemOperand::MOLoad) ||
764 NewMI->mayLoad()) &&
765 "Folded a use to a non-load!");
766 assert(MFI.getObjectOffset(FI) != -1);
767 MachineMemOperand *MMO =
768 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF, FI),
769 F: Flags, Size: MemSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
770 NewMI->addMemOperand(MF, MO: MMO);
771
772 // The pass "x86 speculative load hardening" always attaches symbols to
773 // call instructions. We need copy it form old instruction.
774 NewMI->cloneInstrSymbols(MF, MI);
775
776 return NewMI;
777 }
778
779 // Straight COPY may fold as load/store.
780 if (!isCopyInstr(MI) || Ops.size() != 1)
781 return nullptr;
782
783 const TargetRegisterClass *RC = canFoldCopy(MI, TII: *this, FoldIdx: Ops[0]);
784 if (!RC)
785 return nullptr;
786
787 const MachineOperand &MO = MI.getOperand(i: 1 - Ops[0]);
788 MachineBasicBlock::iterator Pos = MI;
789 if (Flags == MachineMemOperand::MOStore) {
790 if (MO.isUndef()) {
791 // If this is an undef copy, we do not need to bother we inserting spill
792 // code.
793 BuildMI(BB&: *MBB, I: Pos, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::KILL)).add(MO);
794 } else {
795 storeRegToStackSlot(MBB&: *MBB, MI: Pos, SrcReg: MO.getReg(), isKill: MO.isKill(), FrameIndex: FI, RC,
796 VReg: Register());
797 }
798 } else
799 loadRegFromStackSlot(MBB&: *MBB, MI: Pos, DestReg: MO.getReg(), FrameIndex: FI, RC, VReg: Register());
800
801 return &*--Pos;
802}
803
804MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
805 ArrayRef<unsigned> Ops,
806 MachineInstr &LoadMI,
807 LiveIntervals *LIS) const {
808 assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
809#ifndef NDEBUG
810 for (unsigned OpIdx : Ops)
811 assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
812#endif
813
814 MachineBasicBlock &MBB = *MI.getParent();
815 MachineFunction &MF = *MBB.getParent();
816
817 // Ask the target to do the actual folding.
818 MachineInstr *NewMI = nullptr;
819 int FrameIndex = 0;
820
821 if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
822 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
823 MI.getOpcode() == TargetOpcode::STATEPOINT) &&
824 isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
825 // Fold stackmap/patchpoint.
826 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, TII: *this);
827 if (NewMI)
828 NewMI = &*MBB.insert(I: MI, MI: NewMI);
829 } else if (MI.isInlineAsm() && isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
830 return foldInlineAsmMemOperand(MI, Ops, FI: FrameIndex, TII: *this);
831 } else {
832 // Ask the target to do the actual folding.
833 NewMI = foldMemoryOperandImpl(MF, MI, Ops, InsertPt: MI, LoadMI, LIS);
834 }
835
836 if (!NewMI)
837 return nullptr;
838
839 // Copy the memoperands from the load to the folded instruction.
840 if (MI.memoperands_empty()) {
841 NewMI->setMemRefs(MF, MemRefs: LoadMI.memoperands());
842 } else {
843 // Handle the rare case of folding multiple loads.
844 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
845 for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(),
846 E = LoadMI.memoperands_end();
847 I != E; ++I) {
848 NewMI->addMemOperand(MF, MO: *I);
849 }
850 }
851 return NewMI;
852}
853
854/// transferImplicitOperands - MI is a pseudo-instruction, and the lowered
855/// replacement instructions immediately precede it. Copy any implicit
856/// operands from MI to the replacement instruction.
857static void transferImplicitOperands(MachineInstr *MI,
858 const TargetRegisterInfo *TRI) {
859 MachineBasicBlock::iterator CopyMI = MI;
860 --CopyMI;
861
862 Register DstReg = MI->getOperand(i: 0).getReg();
863 for (const MachineOperand &MO : MI->implicit_operands()) {
864 CopyMI->addOperand(Op: MO);
865
866 // Be conservative about preserving kills when subregister defs are
867 // involved. If there was implicit kill of a super-register overlapping the
868 // copy result, we would kill the subregisters previous copies defined.
869
870 if (MO.isKill() && TRI->regsOverlap(RegA: DstReg, RegB: MO.getReg()))
871 CopyMI->getOperand(i: CopyMI->getNumOperands() - 1).setIsKill(false);
872 }
873}
874
875void TargetInstrInfo::lowerCopy(
876 MachineInstr *MI, const TargetRegisterInfo * /*Remove me*/) const {
877 if (MI->allDefsAreDead()) {
878 MI->setDesc(get(Opcode: TargetOpcode::KILL));
879 return;
880 }
881
882 MachineOperand &DstMO = MI->getOperand(i: 0);
883 MachineOperand &SrcMO = MI->getOperand(i: 1);
884
885 bool IdentityCopy = (SrcMO.getReg() == DstMO.getReg());
886 if (IdentityCopy || SrcMO.isUndef()) {
887 // No need to insert an identity copy instruction, but replace with a KILL
888 // if liveness is changed.
889 if (SrcMO.isUndef() || MI->getNumOperands() > 2) {
890 // We must make sure the super-register gets killed. Replace the
891 // instruction with KILL.
892 MI->setDesc(get(Opcode: TargetOpcode::KILL));
893 return;
894 }
895 // Vanilla identity copy.
896 MI->eraseFromParent();
897 return;
898 }
899
900 copyPhysReg(MBB&: *MI->getParent(), MI, DL: MI->getDebugLoc(), DestReg: DstMO.getReg(),
901 SrcReg: SrcMO.getReg(), KillSrc: SrcMO.isKill(),
902 RenamableDest: DstMO.getReg().isPhysical() ? DstMO.isRenamable() : false,
903 RenamableSrc: SrcMO.getReg().isPhysical() ? SrcMO.isRenamable() : false);
904
905 if (MI->getNumOperands() > 2)
906 transferImplicitOperands(MI, TRI: &TRI);
907 MI->eraseFromParent();
908}
909
910bool TargetInstrInfo::hasReassociableOperands(
911 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
912 const MachineOperand &Op1 = Inst.getOperand(i: 1);
913 const MachineOperand &Op2 = Inst.getOperand(i: 2);
914 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
915
916 // We need virtual register definitions for the operands that we will
917 // reassociate.
918 MachineInstr *MI1 = nullptr;
919 MachineInstr *MI2 = nullptr;
920 if (Op1.isReg() && Op1.getReg().isVirtual())
921 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
922 if (Op2.isReg() && Op2.getReg().isVirtual())
923 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
924
925 // And at least one operand must be defined in MBB.
926 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
927}
928
929bool TargetInstrInfo::areOpcodesEqualOrInverse(unsigned Opcode1,
930 unsigned Opcode2) const {
931 return Opcode1 == Opcode2 || getInverseOpcode(Opcode: Opcode1) == Opcode2;
932}
933
934bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
935 bool &Commuted) const {
936 const MachineBasicBlock *MBB = Inst.getParent();
937 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
938 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 1).getReg());
939 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
940 unsigned Opcode = Inst.getOpcode();
941
942 // If only one operand has the same or inverse opcode and it's the second
943 // source operand, the operands must be commuted.
944 Commuted = !areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
945 areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI2->getOpcode());
946 if (Commuted)
947 std::swap(a&: MI1, b&: MI2);
948
949 // 1. The previous instruction must be the same type as Inst.
950 // 2. The previous instruction must also be associative/commutative or be the
951 // inverse of such an operation (this can be different even for
952 // instructions with the same opcode if traits like fast-math-flags are
953 // included).
954 // 3. The previous instruction must have virtual register definitions for its
955 // operands in the same basic block as Inst.
956 // 4. The previous instruction's result must only be used by Inst.
957 return areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
958 (isAssociativeAndCommutative(Inst: *MI1) ||
959 isAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
960 hasReassociableOperands(Inst: *MI1, MBB) &&
961 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
962}
963
964// 1. The operation must be associative and commutative or be the inverse of
965// such an operation.
966// 2. The instruction must have virtual register definitions for its
967// operands in the same basic block.
968// 3. The instruction must have a reassociable sibling.
969bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
970 bool &Commuted) const {
971 return (isAssociativeAndCommutative(Inst) ||
972 isAssociativeAndCommutative(Inst, /* Invert */ true)) &&
973 hasReassociableOperands(Inst, MBB: Inst.getParent()) &&
974 hasReassociableSibling(Inst, Commuted);
975}
976
977// Utility routine that checks if \param MO is defined by an
978// \param CombineOpc instruction in the basic block \param MBB.
979// If \param CombineOpc is not provided, the OpCode check will
980// be skipped.
981static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
982 unsigned CombineOpc = 0) {
983 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
984 MachineInstr *MI = nullptr;
985
986 if (MO.isReg() && MO.getReg().isVirtual())
987 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
988 // And it needs to be in the trace (otherwise, it won't have a depth).
989 if (!MI || MI->getParent() != &MBB ||
990 (MI->getOpcode() != CombineOpc && CombineOpc != 0))
991 return false;
992 // Must only used by the user we combine with.
993 if (!MRI.hasOneNonDBGUse(RegNo: MO.getReg()))
994 return false;
995
996 return true;
997}
998
999// A chain of accumulation instructions will be selected IFF:
1000// 1. All the accumulation instructions in the chain have the same opcode,
1001// besides the first that has a slightly different opcode because it does
1002// not accumulate into a register.
1003// 2. All the instructions in the chain are combinable (have a single use
1004// which itself is part of the chain).
1005// 3. Meets the required minimum length.
1006void TargetInstrInfo::getAccumulatorChain(
1007 MachineInstr *CurrentInstr, SmallVectorImpl<Register> &Chain) const {
1008 // Walk up the chain of accumulation instructions and collect them in the
1009 // vector.
1010 MachineBasicBlock &MBB = *CurrentInstr->getParent();
1011 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1012 unsigned AccumulatorOpcode = CurrentInstr->getOpcode();
1013 std::optional<unsigned> ChainStartOpCode =
1014 getAccumulationStartOpcode(Opcode: AccumulatorOpcode);
1015
1016 if (!ChainStartOpCode.has_value())
1017 return;
1018
1019 // Push the first accumulator result to the start of the chain.
1020 Chain.push_back(Elt: CurrentInstr->getOperand(i: 0).getReg());
1021
1022 // Collect the accumulator input register from all instructions in the chain.
1023 while (CurrentInstr &&
1024 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1), CombineOpc: AccumulatorOpcode)) {
1025 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1026 CurrentInstr = MRI.getUniqueVRegDef(Reg: CurrentInstr->getOperand(i: 1).getReg());
1027 }
1028
1029 // Add the instruction at the top of the chain.
1030 if (CurrentInstr->getOpcode() == AccumulatorOpcode &&
1031 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1)))
1032 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1033}
1034
1035/// Find chains of accumulations that can be rewritten as a tree for increased
1036/// ILP.
1037bool TargetInstrInfo::getAccumulatorReassociationPatterns(
1038 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns) const {
1039 if (!EnableAccReassociation)
1040 return false;
1041
1042 unsigned Opc = Root.getOpcode();
1043 if (!isAccumulationOpcode(Opcode: Opc))
1044 return false;
1045
1046 // Verify that this is the end of the chain.
1047 MachineBasicBlock &MBB = *Root.getParent();
1048 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1049 if (!MRI.hasOneNonDBGUser(RegNo: Root.getOperand(i: 0).getReg()))
1050 return false;
1051
1052 auto User = MRI.use_instr_begin(RegNo: Root.getOperand(i: 0).getReg());
1053 if (User->getOpcode() == Opc)
1054 return false;
1055
1056 // Walk up the use chain and collect the reduction chain.
1057 SmallVector<Register, 32> Chain;
1058 getAccumulatorChain(CurrentInstr: &Root, Chain);
1059
1060 // Reject chains which are too short to be worth modifying.
1061 if (Chain.size() < MinAccumulatorDepth)
1062 return false;
1063
1064 // Check if the MBB this instruction is a part of contains any other chains.
1065 // If so, don't apply it.
1066 SmallSet<Register, 32> ReductionChain(llvm::from_range, Chain);
1067 for (const auto &I : MBB) {
1068 if (I.getOpcode() == Opc &&
1069 !ReductionChain.contains(V: I.getOperand(i: 0).getReg()))
1070 return false;
1071 }
1072
1073 Patterns.push_back(Elt: MachineCombinerPattern::ACC_CHAIN);
1074 return true;
1075}
1076
1077// Reduce branches of the accumulator tree by adding them together.
1078void TargetInstrInfo::reduceAccumulatorTree(
1079 SmallVectorImpl<Register> &RegistersToReduce,
1080 SmallVectorImpl<MachineInstr *> &InsInstrs, MachineFunction &MF,
1081 MachineInstr &Root, MachineRegisterInfo &MRI,
1082 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
1083 Register ResultReg) const {
1084 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1085 SmallVector<Register, 8> NewRegs;
1086
1087 // Get the opcode for the reduction instruction we will need to build.
1088 // If for some reason it is not defined, early exit and don't apply this.
1089 unsigned ReduceOpCode = getReduceOpcodeForAccumulator(AccumulatorOpCode: Root.getOpcode());
1090
1091 for (unsigned int i = 1; i <= (RegistersToReduce.size() / 2); i += 2) {
1092 auto RHS = RegistersToReduce[i - 1];
1093 auto LHS = RegistersToReduce[i];
1094 Register Dest;
1095 // If we are reducing 2 registers, reuse the original result register.
1096 if (RegistersToReduce.size() == 2)
1097 Dest = ResultReg;
1098 // Otherwise, create a new virtual register to hold the partial sum.
1099 else {
1100 auto NewVR = MRI.createVirtualRegister(
1101 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()));
1102 Dest = NewVR;
1103 NewRegs.push_back(Elt: Dest);
1104 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: Dest, y: InsInstrs.size()));
1105 }
1106
1107 // Create the new reduction instruction.
1108 MachineInstrBuilder MIB =
1109 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: ReduceOpCode), DestReg: Dest)
1110 .addReg(RegNo: RHS, Flags: getKillRegState(B: true))
1111 .addReg(RegNo: LHS, Flags: getKillRegState(B: true));
1112 // Copy any flags needed from the original instruction.
1113 MIB->setFlags(Root.getFlags());
1114 InsInstrs.push_back(Elt: MIB);
1115 }
1116
1117 // If the number of registers to reduce is odd, add the remaining register to
1118 // the vector of registers to reduce.
1119 if (RegistersToReduce.size() % 2 != 0)
1120 NewRegs.push_back(Elt: RegistersToReduce[RegistersToReduce.size() - 1]);
1121
1122 RegistersToReduce = std::move(NewRegs);
1123}
1124
1125// The concept of the reassociation pass is that these operations can benefit
1126// from this kind of transformation:
1127//
1128// A = ? op ?
1129// B = A op X (Prev)
1130// C = B op Y (Root)
1131// -->
1132// A = ? op ?
1133// B = X op Y
1134// C = A op B
1135//
1136// breaking the dependency between A and B, allowing them to be executed in
1137// parallel (or back-to-back in a pipeline) instead of depending on each other.
1138
1139// FIXME: This has the potential to be expensive (compile time) while not
1140// improving the code at all. Some ways to limit the overhead:
1141// 1. Track successful transforms; bail out if hit rate gets too low.
1142// 2. Only enable at -O3 or some other non-default optimization level.
1143// 3. Pre-screen pattern candidates here: if an operand of the previous
1144// instruction is known to not increase the critical path, then don't match
1145// that pattern.
1146bool TargetInstrInfo::getMachineCombinerPatterns(
1147 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
1148 bool DoRegPressureReduce) const {
1149 bool Commute;
1150 if (isReassociationCandidate(Inst: Root, Commuted&: Commute)) {
1151 // We found a sequence of instructions that may be suitable for a
1152 // reassociation of operands to increase ILP. Specify each commutation
1153 // possibility for the Prev instruction in the sequence and let the
1154 // machine combiner decide if changing the operands is worthwhile.
1155 if (Commute) {
1156 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_YB);
1157 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_YB);
1158 } else {
1159 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_BY);
1160 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_BY);
1161 }
1162 return true;
1163 }
1164 if (getAccumulatorReassociationPatterns(Root, Patterns))
1165 return true;
1166
1167 return false;
1168}
1169
1170/// Return true when a code sequence can improve loop throughput.
1171bool TargetInstrInfo::isThroughputPattern(unsigned Pattern) const {
1172 return false;
1173}
1174
1175CombinerObjective
1176TargetInstrInfo::getCombinerObjective(unsigned Pattern) const {
1177 switch (Pattern) {
1178 case MachineCombinerPattern::ACC_CHAIN:
1179 return CombinerObjective::MustReduceDepth;
1180 default:
1181 return CombinerObjective::Default;
1182 }
1183}
1184
1185std::pair<unsigned, unsigned>
1186TargetInstrInfo::getReassociationOpcodes(unsigned Pattern,
1187 const MachineInstr &Root,
1188 const MachineInstr &Prev) const {
1189 bool AssocCommutRoot = isAssociativeAndCommutative(Inst: Root);
1190 bool AssocCommutPrev = isAssociativeAndCommutative(Inst: Prev);
1191
1192 // Early exit if both opcodes are associative and commutative. It's a trivial
1193 // reassociation when we only change operands order. In this case opcodes are
1194 // not required to have inverse versions.
1195 if (AssocCommutRoot && AssocCommutPrev) {
1196 assert(Root.getOpcode() == Prev.getOpcode() && "Expected to be equal");
1197 return std::make_pair(x: Root.getOpcode(), y: Root.getOpcode());
1198 }
1199
1200 // At least one instruction is not associative or commutative.
1201 // Since we have matched one of the reassociation patterns, we expect that the
1202 // instructions' opcodes are equal or one of them is the inversion of the
1203 // other.
1204 assert(areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()) &&
1205 "Incorrectly matched pattern");
1206 unsigned AssocCommutOpcode = Root.getOpcode();
1207 unsigned InverseOpcode = *getInverseOpcode(Opcode: Root.getOpcode());
1208 if (!AssocCommutRoot)
1209 std::swap(a&: AssocCommutOpcode, b&: InverseOpcode);
1210
1211 // The transformation rule (`+` is any associative and commutative binary
1212 // operation, `-` is the inverse):
1213 // REASSOC_AX_BY:
1214 // (A + X) + Y => A + (X + Y)
1215 // (A + X) - Y => A + (X - Y)
1216 // (A - X) + Y => A - (X - Y)
1217 // (A - X) - Y => A - (X + Y)
1218 // REASSOC_XA_BY:
1219 // (X + A) + Y => (X + Y) + A
1220 // (X + A) - Y => (X - Y) + A
1221 // (X - A) + Y => (X + Y) - A
1222 // (X - A) - Y => (X - Y) - A
1223 // REASSOC_AX_YB:
1224 // Y + (A + X) => (Y + X) + A
1225 // Y - (A + X) => (Y - X) - A
1226 // Y + (A - X) => (Y - X) + A
1227 // Y - (A - X) => (Y + X) - A
1228 // REASSOC_XA_YB:
1229 // Y + (X + A) => (Y + X) + A
1230 // Y - (X + A) => (Y - X) - A
1231 // Y + (X - A) => (Y + X) - A
1232 // Y - (X - A) => (Y - X) + A
1233 switch (Pattern) {
1234 default:
1235 llvm_unreachable("Unexpected pattern");
1236 case MachineCombinerPattern::REASSOC_AX_BY:
1237 if (!AssocCommutRoot && AssocCommutPrev)
1238 return {AssocCommutOpcode, InverseOpcode};
1239 if (AssocCommutRoot && !AssocCommutPrev)
1240 return {InverseOpcode, InverseOpcode};
1241 if (!AssocCommutRoot && !AssocCommutPrev)
1242 return {InverseOpcode, AssocCommutOpcode};
1243 break;
1244 case MachineCombinerPattern::REASSOC_XA_BY:
1245 if (!AssocCommutRoot && AssocCommutPrev)
1246 return {AssocCommutOpcode, InverseOpcode};
1247 if (AssocCommutRoot && !AssocCommutPrev)
1248 return {InverseOpcode, AssocCommutOpcode};
1249 if (!AssocCommutRoot && !AssocCommutPrev)
1250 return {InverseOpcode, InverseOpcode};
1251 break;
1252 case MachineCombinerPattern::REASSOC_AX_YB:
1253 if (!AssocCommutRoot && AssocCommutPrev)
1254 return {InverseOpcode, InverseOpcode};
1255 if (AssocCommutRoot && !AssocCommutPrev)
1256 return {AssocCommutOpcode, InverseOpcode};
1257 if (!AssocCommutRoot && !AssocCommutPrev)
1258 return {InverseOpcode, AssocCommutOpcode};
1259 break;
1260 case MachineCombinerPattern::REASSOC_XA_YB:
1261 if (!AssocCommutRoot && AssocCommutPrev)
1262 return {InverseOpcode, InverseOpcode};
1263 if (AssocCommutRoot && !AssocCommutPrev)
1264 return {InverseOpcode, AssocCommutOpcode};
1265 if (!AssocCommutRoot && !AssocCommutPrev)
1266 return {AssocCommutOpcode, InverseOpcode};
1267 break;
1268 }
1269 llvm_unreachable("Unhandled combination");
1270}
1271
1272// Return a pair of boolean flags showing if the new root and new prev operands
1273// must be swapped. See visual example of the rule in
1274// TargetInstrInfo::getReassociationOpcodes.
1275static std::pair<bool, bool> mustSwapOperands(unsigned Pattern) {
1276 switch (Pattern) {
1277 default:
1278 llvm_unreachable("Unexpected pattern");
1279 case MachineCombinerPattern::REASSOC_AX_BY:
1280 return {false, false};
1281 case MachineCombinerPattern::REASSOC_XA_BY:
1282 return {true, false};
1283 case MachineCombinerPattern::REASSOC_AX_YB:
1284 return {true, true};
1285 case MachineCombinerPattern::REASSOC_XA_YB:
1286 return {true, true};
1287 }
1288}
1289
1290void TargetInstrInfo::getReassociateOperandIndices(
1291 const MachineInstr &Root, unsigned Pattern,
1292 std::array<unsigned, 5> &OperandIndices) const {
1293 switch (Pattern) {
1294 case MachineCombinerPattern::REASSOC_AX_BY:
1295 OperandIndices = {1, 1, 1, 2, 2};
1296 break;
1297 case MachineCombinerPattern::REASSOC_AX_YB:
1298 OperandIndices = {2, 1, 2, 2, 1};
1299 break;
1300 case MachineCombinerPattern::REASSOC_XA_BY:
1301 OperandIndices = {1, 2, 1, 1, 2};
1302 break;
1303 case MachineCombinerPattern::REASSOC_XA_YB:
1304 OperandIndices = {2, 2, 2, 1, 1};
1305 break;
1306 default:
1307 llvm_unreachable("unexpected MachineCombinerPattern");
1308 }
1309}
1310
1311/// Attempt the reassociation transformation to reduce critical path length.
1312/// See the above comments before getMachineCombinerPatterns().
1313void TargetInstrInfo::reassociateOps(
1314 MachineInstr &Root, MachineInstr &Prev, unsigned Pattern,
1315 SmallVectorImpl<MachineInstr *> &InsInstrs,
1316 SmallVectorImpl<MachineInstr *> &DelInstrs,
1317 ArrayRef<unsigned> OperandIndices,
1318 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
1319 MachineFunction *MF = Root.getMF();
1320 MachineRegisterInfo &MRI = MF->getRegInfo();
1321 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1322 const TargetRegisterClass *RC = Root.getRegClassConstraint(OpIdx: 0, TII, TRI: &TRI);
1323
1324 MachineOperand &OpA = Prev.getOperand(i: OperandIndices[1]);
1325 MachineOperand &OpB = Root.getOperand(i: OperandIndices[2]);
1326 MachineOperand &OpX = Prev.getOperand(i: OperandIndices[3]);
1327 MachineOperand &OpY = Root.getOperand(i: OperandIndices[4]);
1328 MachineOperand &OpC = Root.getOperand(i: 0);
1329
1330 Register RegA = OpA.getReg();
1331 unsigned SubRegA = OpA.getSubReg();
1332 Register RegB = OpB.getReg();
1333 Register RegX = OpX.getReg();
1334 unsigned SubRegX = OpX.getSubReg();
1335 Register RegY = OpY.getReg();
1336 unsigned SubRegY = OpY.getSubReg();
1337 Register RegC = OpC.getReg();
1338
1339 if (RegA.isVirtual())
1340 MRI.constrainRegClass(Reg: RegA, RC);
1341 if (RegB.isVirtual())
1342 MRI.constrainRegClass(Reg: RegB, RC);
1343 if (RegX.isVirtual())
1344 MRI.constrainRegClass(Reg: RegX, RC);
1345 if (RegY.isVirtual())
1346 MRI.constrainRegClass(Reg: RegY, RC);
1347 if (RegC.isVirtual())
1348 MRI.constrainRegClass(Reg: RegC, RC);
1349
1350 // Create a new virtual register for the result of (X op Y) instead of
1351 // recycling RegB because the MachineCombiner's computation of the critical
1352 // path requires a new register definition rather than an existing one.
1353 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
1354 unsigned SubRegNewVR = 0;
1355 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
1356
1357 auto [NewRootOpc, NewPrevOpc] = getReassociationOpcodes(Pattern, Root, Prev);
1358 bool KillA = OpA.isKill();
1359 bool KillX = OpX.isKill();
1360 bool KillY = OpY.isKill();
1361 bool KillNewVR = true;
1362
1363 auto [SwapRootOperands, SwapPrevOperands] = mustSwapOperands(Pattern);
1364
1365 if (SwapPrevOperands) {
1366 std::swap(a&: RegX, b&: RegY);
1367 std::swap(a&: SubRegX, b&: SubRegY);
1368 std::swap(a&: KillX, b&: KillY);
1369 }
1370
1371 unsigned PrevFirstOpIdx, PrevSecondOpIdx;
1372 unsigned RootFirstOpIdx, RootSecondOpIdx;
1373 switch (Pattern) {
1374 case MachineCombinerPattern::REASSOC_AX_BY:
1375 PrevFirstOpIdx = OperandIndices[1];
1376 PrevSecondOpIdx = OperandIndices[3];
1377 RootFirstOpIdx = OperandIndices[2];
1378 RootSecondOpIdx = OperandIndices[4];
1379 break;
1380 case MachineCombinerPattern::REASSOC_AX_YB:
1381 PrevFirstOpIdx = OperandIndices[1];
1382 PrevSecondOpIdx = OperandIndices[3];
1383 RootFirstOpIdx = OperandIndices[4];
1384 RootSecondOpIdx = OperandIndices[2];
1385 break;
1386 case MachineCombinerPattern::REASSOC_XA_BY:
1387 PrevFirstOpIdx = OperandIndices[3];
1388 PrevSecondOpIdx = OperandIndices[1];
1389 RootFirstOpIdx = OperandIndices[2];
1390 RootSecondOpIdx = OperandIndices[4];
1391 break;
1392 case MachineCombinerPattern::REASSOC_XA_YB:
1393 PrevFirstOpIdx = OperandIndices[3];
1394 PrevSecondOpIdx = OperandIndices[1];
1395 RootFirstOpIdx = OperandIndices[4];
1396 RootSecondOpIdx = OperandIndices[2];
1397 break;
1398 default:
1399 llvm_unreachable("unexpected MachineCombinerPattern");
1400 }
1401
1402 // Basically BuildMI but doesn't add implicit operands by default.
1403 auto buildMINoImplicit = [](MachineFunction &MF, const MIMetadata &MIMD,
1404 const MCInstrDesc &MCID, Register DestReg) {
1405 return MachineInstrBuilder(
1406 MF, MF.CreateMachineInstr(MCID, DL: MIMD.getDL(), /*NoImpl=*/NoImplicit: true))
1407 .copyMIMetadata(MIMD)
1408 .addReg(RegNo: DestReg, Flags: RegState::Define);
1409 };
1410
1411 // Create new instructions for insertion.
1412 MachineInstrBuilder MIB1 =
1413 buildMINoImplicit(*MF, MIMetadata(Prev), TII->get(Opcode: NewPrevOpc), NewVR);
1414 for (const auto &MO : Prev.explicit_operands()) {
1415 unsigned Idx = MO.getOperandNo();
1416 // Skip the result operand we'd already added.
1417 if (Idx == 0)
1418 continue;
1419 if (Idx == PrevFirstOpIdx)
1420 MIB1.addReg(RegNo: RegX, Flags: getKillRegState(B: KillX), SubReg: SubRegX);
1421 else if (Idx == PrevSecondOpIdx)
1422 MIB1.addReg(RegNo: RegY, Flags: getKillRegState(B: KillY), SubReg: SubRegY);
1423 else
1424 MIB1.add(MO);
1425 }
1426 MIB1.copyImplicitOps(OtherMI: Prev);
1427
1428 if (SwapRootOperands) {
1429 std::swap(a&: RegA, b&: NewVR);
1430 std::swap(a&: SubRegA, b&: SubRegNewVR);
1431 std::swap(a&: KillA, b&: KillNewVR);
1432 }
1433
1434 MachineInstrBuilder MIB2 =
1435 buildMINoImplicit(*MF, MIMetadata(Root), TII->get(Opcode: NewRootOpc), RegC);
1436 for (const auto &MO : Root.explicit_operands()) {
1437 unsigned Idx = MO.getOperandNo();
1438 // Skip the result operand.
1439 if (Idx == 0)
1440 continue;
1441 if (Idx == RootFirstOpIdx)
1442 MIB2 = MIB2.addReg(RegNo: RegA, Flags: getKillRegState(B: KillA), SubReg: SubRegA);
1443 else if (Idx == RootSecondOpIdx)
1444 MIB2 = MIB2.addReg(RegNo: NewVR, Flags: getKillRegState(B: KillNewVR), SubReg: SubRegNewVR);
1445 else
1446 MIB2 = MIB2.add(MO);
1447 }
1448 MIB2.copyImplicitOps(OtherMI: Root);
1449
1450 // Propagate FP flags from the original instructions.
1451 // But clear poison-generating flags because those may not be valid now.
1452 // TODO: There should be a helper function for copying only fast-math-flags.
1453 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
1454 MIB1->setFlags(IntersectedFlags);
1455 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1456 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1457 MIB1->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1458 MIB1->clearFlag(Flag: MachineInstr::MIFlag::Disjoint);
1459
1460 MIB2->setFlags(IntersectedFlags);
1461 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1462 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1463 MIB2->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1464 MIB2->clearFlag(Flag: MachineInstr::MIFlag::Disjoint);
1465
1466 setSpecialOperandAttr(OldMI1&: Root, OldMI2&: Prev, NewMI1&: *MIB1, NewMI2&: *MIB2);
1467
1468 // Record new instructions for insertion and old instructions for deletion.
1469 InsInstrs.push_back(Elt: MIB1);
1470 InsInstrs.push_back(Elt: MIB2);
1471 DelInstrs.push_back(Elt: &Prev);
1472 DelInstrs.push_back(Elt: &Root);
1473
1474 // We transformed:
1475 // B = A op X (Prev)
1476 // C = B op Y (Root)
1477 // Into:
1478 // B = X op Y (MIB1)
1479 // C = A op B (MIB2)
1480 // C has the same value as before, B doesn't; as such, keep the debug number
1481 // of C but not of B.
1482 if (unsigned OldRootNum = Root.peekDebugInstrNum())
1483 MIB2.getInstr()->setDebugInstrNum(OldRootNum);
1484}
1485
1486void TargetInstrInfo::genAlternativeCodeSequence(
1487 MachineInstr &Root, unsigned Pattern,
1488 SmallVectorImpl<MachineInstr *> &InsInstrs,
1489 SmallVectorImpl<MachineInstr *> &DelInstrs,
1490 DenseMap<Register, unsigned> &InstIdxForVirtReg) const {
1491 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1492 MachineBasicBlock &MBB = *Root.getParent();
1493 MachineFunction &MF = *MBB.getParent();
1494 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1495
1496 switch (Pattern) {
1497 case MachineCombinerPattern::REASSOC_AX_BY:
1498 case MachineCombinerPattern::REASSOC_AX_YB:
1499 case MachineCombinerPattern::REASSOC_XA_BY:
1500 case MachineCombinerPattern::REASSOC_XA_YB: {
1501 // Select the previous instruction in the sequence based on the input
1502 // pattern.
1503 std::array<unsigned, 5> OperandIndices;
1504 getReassociateOperandIndices(Root, Pattern, OperandIndices);
1505 MachineInstr *Prev =
1506 MRI.getUniqueVRegDef(Reg: Root.getOperand(i: OperandIndices[0]).getReg());
1507
1508 // Don't reassociate if Prev and Root are in different blocks.
1509 if (Prev->getParent() != Root.getParent())
1510 return;
1511
1512 reassociateOps(Root, Prev&: *Prev, Pattern, InsInstrs, DelInstrs, OperandIndices,
1513 InstrIdxForVirtReg&: InstIdxForVirtReg);
1514 break;
1515 }
1516 case MachineCombinerPattern::ACC_CHAIN: {
1517 SmallVector<Register, 32> ChainRegs;
1518 getAccumulatorChain(CurrentInstr: &Root, Chain&: ChainRegs);
1519 unsigned int Depth = ChainRegs.size();
1520 assert(MaxAccumulatorWidth > 1 &&
1521 "Max accumulator width set to illegal value");
1522 unsigned int MaxWidth = Log2_32(Value: Depth) < MaxAccumulatorWidth
1523 ? Log2_32(Value: Depth)
1524 : MaxAccumulatorWidth;
1525
1526 // Walk down the chain and rewrite it as a tree.
1527 for (auto IndexedReg : llvm::enumerate(First: llvm::reverse(C&: ChainRegs))) {
1528 // No need to rewrite the first node, it is already perfect as it is.
1529 if (IndexedReg.index() == 0)
1530 continue;
1531
1532 // FIXME: Losing subregisters
1533 MachineInstr *Instr = MRI.getUniqueVRegDef(Reg: IndexedReg.value());
1534 MachineInstrBuilder MIB;
1535 Register AccReg;
1536 if (IndexedReg.index() < MaxWidth) {
1537 // Now we need to create new instructions for the first row.
1538 AccReg = Instr->getOperand(i: 0).getReg();
1539 unsigned OpCode = getAccumulationStartOpcode(Opcode: Root.getOpcode());
1540
1541 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: OpCode), DestReg: AccReg)
1542 .addReg(RegNo: Instr->getOperand(i: 2).getReg(),
1543 Flags: getKillRegState(B: Instr->getOperand(i: 2).isKill()))
1544 .addReg(RegNo: Instr->getOperand(i: 3).getReg(),
1545 Flags: getKillRegState(B: Instr->getOperand(i: 3).isKill()));
1546 } else {
1547 // For the remaining cases, we need to use an output register of one of
1548 // the newly inserted instuctions as operand 1
1549 AccReg = Instr->getOperand(i: 0).getReg() == Root.getOperand(i: 0).getReg()
1550 ? MRI.createVirtualRegister(
1551 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()))
1552 : Instr->getOperand(i: 0).getReg();
1553 assert(IndexedReg.index() >= MaxWidth);
1554 auto AccumulatorInput =
1555 ChainRegs[Depth - (IndexedReg.index() - MaxWidth) - 1];
1556 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: Instr->getOpcode()),
1557 DestReg: AccReg)
1558 .addReg(RegNo: AccumulatorInput, Flags: getKillRegState(B: true))
1559 .addReg(RegNo: Instr->getOperand(i: 2).getReg(),
1560 Flags: getKillRegState(B: Instr->getOperand(i: 2).isKill()))
1561 .addReg(RegNo: Instr->getOperand(i: 3).getReg(),
1562 Flags: getKillRegState(B: Instr->getOperand(i: 3).isKill()));
1563 }
1564
1565 MIB->setFlags(Instr->getFlags());
1566 InstIdxForVirtReg.insert(KV: std::make_pair(x&: AccReg, y: InsInstrs.size()));
1567 InsInstrs.push_back(Elt: MIB);
1568 DelInstrs.push_back(Elt: Instr);
1569 }
1570
1571 SmallVector<Register, 8> RegistersToReduce;
1572 for (unsigned i = (InsInstrs.size() - MaxWidth); i < InsInstrs.size();
1573 ++i) {
1574 auto Reg = InsInstrs[i]->getOperand(i: 0).getReg();
1575 RegistersToReduce.push_back(Elt: Reg);
1576 }
1577
1578 while (RegistersToReduce.size() > 1)
1579 reduceAccumulatorTree(RegistersToReduce, InsInstrs, MF, Root, MRI,
1580 InstrIdxForVirtReg&: InstIdxForVirtReg, ResultReg: Root.getOperand(i: 0).getReg());
1581
1582 break;
1583 }
1584 }
1585}
1586
1587MachineTraceStrategy TargetInstrInfo::getMachineCombinerTraceStrategy() const {
1588 return MachineTraceStrategy::TS_MinInstrCount;
1589}
1590
1591bool TargetInstrInfo::isReMaterializableImpl(
1592 const MachineInstr &MI) const {
1593 const MachineFunction &MF = *MI.getMF();
1594 const MachineRegisterInfo &MRI = MF.getRegInfo();
1595
1596 // Remat clients assume operand 0 is the defined register.
1597 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
1598 return false;
1599 Register DefReg = MI.getOperand(i: 0).getReg();
1600
1601 // A sub-register definition can only be rematerialized if the instruction
1602 // doesn't read the other parts of the register. Otherwise it is really a
1603 // read-modify-write operation on the full virtual register which cannot be
1604 // moved safely.
1605 if (DefReg.isVirtual() && MI.getOperand(i: 0).getSubReg() &&
1606 MI.readsVirtualRegister(Reg: DefReg))
1607 return false;
1608
1609 // A load from a fixed stack slot can be rematerialized. This may be
1610 // redundant with subsequent checks, but it's target-independent,
1611 // simple, and a common case.
1612 int FrameIdx = 0;
1613 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
1614 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
1615 return true;
1616
1617 // Avoid instructions obviously unsafe for remat.
1618 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
1619 MI.hasUnmodeledSideEffects())
1620 return false;
1621
1622 // Don't remat inline asm. We have no idea how expensive it is
1623 // even if it's side effect free.
1624 if (MI.isInlineAsm())
1625 return false;
1626
1627 // Avoid instructions which load from potentially varying memory.
1628 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
1629 return false;
1630
1631 // If any of the registers accessed are non-constant, conservatively assume
1632 // the instruction is not rematerializable.
1633 for (const MachineOperand &MO : MI.operands()) {
1634 if (!MO.isReg()) continue;
1635 Register Reg = MO.getReg();
1636 if (Reg == 0)
1637 continue;
1638
1639 // Check for a well-behaved physical register.
1640 if (Reg.isPhysical()) {
1641 if (MO.isUse()) {
1642 // If the physreg has no defs anywhere, it's just an ambient register
1643 // and we can freely move its uses. Alternatively, if it's allocatable,
1644 // it could get allocated to something with a def during allocation.
1645 if (!MRI.isConstantPhysReg(PhysReg: Reg))
1646 return false;
1647 } else {
1648 // A physreg def. We can't remat it.
1649 return false;
1650 }
1651 continue;
1652 }
1653
1654 // Only allow one virtual-register def. There may be multiple defs of the
1655 // same virtual register, though.
1656 if (MO.isDef() && Reg != DefReg)
1657 return false;
1658 }
1659
1660 // Everything checked out.
1661 return true;
1662}
1663
1664int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
1665 const MachineFunction *MF = MI.getMF();
1666 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
1667 bool StackGrowsDown =
1668 TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
1669
1670 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
1671 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
1672
1673 if (!isFrameInstr(I: MI))
1674 return 0;
1675
1676 int SPAdj = TFI->alignSPAdjust(SPAdj: getFrameSize(I: MI));
1677
1678 if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
1679 (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
1680 SPAdj = -SPAdj;
1681
1682 return SPAdj;
1683}
1684
1685/// isSchedulingBoundary - Test if the given instruction should be
1686/// considered a scheduling boundary. This primarily includes labels
1687/// and terminators.
1688bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1689 const MachineBasicBlock *MBB,
1690 const MachineFunction &MF) const {
1691 // Terminators and labels can't be scheduled around.
1692 if (MI.isTerminator() || MI.isPosition())
1693 return true;
1694
1695 // INLINEASM_BR can jump to another block
1696 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1697 return true;
1698
1699 // Don't attempt to schedule around any instruction that defines
1700 // a stack-oriented pointer, as it's unlikely to be profitable. This
1701 // saves compile time, because it doesn't require every single
1702 // stack slot reference to depend on the instruction that does the
1703 // modification.
1704 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
1705 return MI.modifiesRegister(Reg: TLI.getStackPointerRegisterToSaveRestore(), TRI: &TRI);
1706}
1707
1708// Provide a global flag for disabling the PreRA hazard recognizer that targets
1709// may choose to honor.
1710bool TargetInstrInfo::usePreRAHazardRecognizer() const {
1711 return !DisableHazardRecognizer;
1712}
1713
1714// Default implementation of CreateTargetRAHazardRecognizer.
1715ScheduleHazardRecognizer *TargetInstrInfo::
1716CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
1717 const ScheduleDAG *DAG) const {
1718 // Dummy hazard recognizer allows all instructions to issue.
1719 return new ScheduleHazardRecognizer();
1720}
1721
1722// Default implementation of CreateTargetMIHazardRecognizer.
1723ScheduleHazardRecognizer *TargetInstrInfo::CreateTargetMIHazardRecognizer(
1724 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
1725 return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler");
1726}
1727
1728// Default implementation of CreateTargetPostRAHazardRecognizer.
1729ScheduleHazardRecognizer *TargetInstrInfo::
1730CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1731 const ScheduleDAG *DAG) const {
1732 return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1733}
1734
1735// Default implementation of getMemOperandWithOffset.
1736bool TargetInstrInfo::getMemOperandWithOffset(
1737 const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
1738 bool &OffsetIsScalable, const TargetRegisterInfo * /*RemoveMe*/) const {
1739 SmallVector<const MachineOperand *, 4> BaseOps;
1740 LocationSize Width = LocationSize::precise(Value: 0);
1741 if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
1742 Width, TRI: &TRI) ||
1743 BaseOps.size() != 1)
1744 return false;
1745 BaseOp = BaseOps.front();
1746 return true;
1747}
1748
1749//===----------------------------------------------------------------------===//
1750// SelectionDAG latency interface.
1751//===----------------------------------------------------------------------===//
1752
1753std::optional<unsigned>
1754TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1755 SDNode *DefNode, unsigned DefIdx,
1756 SDNode *UseNode, unsigned UseIdx) const {
1757 if (!ItinData || ItinData->isEmpty())
1758 return std::nullopt;
1759
1760 if (!DefNode->isMachineOpcode())
1761 return std::nullopt;
1762
1763 unsigned DefClass = get(Opcode: DefNode->getMachineOpcode()).getSchedClass();
1764 if (!UseNode->isMachineOpcode())
1765 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1766 unsigned UseClass = get(Opcode: UseNode->getMachineOpcode()).getSchedClass();
1767 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1768}
1769
1770unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1771 SDNode *N) const {
1772 if (!ItinData || ItinData->isEmpty())
1773 return 1;
1774
1775 if (!N->isMachineOpcode())
1776 return 1;
1777
1778 return ItinData->getStageLatency(ItinClassIndx: get(Opcode: N->getMachineOpcode()).getSchedClass());
1779}
1780
1781//===----------------------------------------------------------------------===//
1782// MachineInstr latency interface.
1783//===----------------------------------------------------------------------===//
1784
1785unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
1786 const MachineInstr &MI) const {
1787 if (!ItinData || ItinData->isEmpty())
1788 return 1;
1789
1790 unsigned Class = MI.getDesc().getSchedClass();
1791 int UOps = ItinData->Itineraries[Class].NumMicroOps;
1792 if (UOps >= 0)
1793 return UOps;
1794
1795 // The # of u-ops is dynamically determined. The specific target should
1796 // override this function to return the right number.
1797 return 1;
1798}
1799
1800/// Return the default expected latency for a def based on it's opcode.
1801unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel,
1802 const MachineInstr &DefMI) const {
1803 if (DefMI.isTransient())
1804 return 0;
1805 if (DefMI.mayLoad())
1806 return SchedModel.LoadLatency;
1807 if (isHighLatencyDef(opc: DefMI.getOpcode()))
1808 return SchedModel.HighLatency;
1809 return 1;
1810}
1811
1812unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const {
1813 return 0;
1814}
1815
1816unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1817 const MachineInstr &MI,
1818 unsigned *PredCost) const {
1819 // Default to one cycle for no itinerary. However, an "empty" itinerary may
1820 // still have a MinLatency property, which getStageLatency checks.
1821 if (!ItinData)
1822 return MI.mayLoad() ? 2 : 1;
1823
1824 return ItinData->getStageLatency(ItinClassIndx: MI.getDesc().getSchedClass());
1825}
1826
1827bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
1828 const MachineInstr &DefMI,
1829 unsigned DefIdx) const {
1830 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1831 if (!ItinData || ItinData->isEmpty())
1832 return false;
1833
1834 unsigned DefClass = DefMI.getDesc().getSchedClass();
1835 std::optional<unsigned> DefCycle =
1836 ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1837 return DefCycle && DefCycle <= 1U;
1838}
1839
1840bool TargetInstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
1841 // TODO: We don't split functions where a section attribute has been set
1842 // since the split part may not be placed in a contiguous region. It may also
1843 // be more beneficial to augment the linker to ensure contiguous layout of
1844 // split functions within the same section as specified by the attribute.
1845 if (MF.getFunction().hasSection())
1846 return false;
1847
1848 // We don't want to proceed further for cold functions
1849 // or functions of unknown hotness. Lukewarm functions have no prefix.
1850 std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
1851 if (SectionPrefix &&
1852 (*SectionPrefix == "unlikely" || *SectionPrefix == "unknown")) {
1853 return false;
1854 }
1855
1856 return true;
1857}
1858
1859std::optional<ParamLoadedValue>
1860TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
1861 Register Reg) const {
1862 const MachineFunction *MF = MI.getMF();
1863 DIExpression *Expr = DIExpression::get(Context&: MF->getFunction().getContext(), Elements: {});
1864 int64_t Offset;
1865 bool OffsetIsScalable;
1866
1867 // To simplify the sub-register handling, verify that we only need to
1868 // consider physical registers.
1869 assert(MF->getProperties().hasNoVRegs());
1870
1871 if (auto DestSrc = isCopyInstr(MI)) {
1872 Register DestReg = DestSrc->Destination->getReg();
1873
1874 // If the copy destination is the forwarding reg, describe the forwarding
1875 // reg using the copy source as the backup location. Example:
1876 //
1877 // x0 = MOV x7
1878 // call callee(x0) ; x0 described as x7
1879 if (Reg == DestReg)
1880 return ParamLoadedValue(*DestSrc->Source, Expr);
1881
1882 // If the target's hook couldn't describe this copy, give up.
1883 return std::nullopt;
1884 } else if (auto RegImm = isAddImmediate(MI, Reg)) {
1885 Register SrcReg = RegImm->Reg;
1886 Offset = RegImm->Imm;
1887 Expr = DIExpression::prepend(Expr, Flags: DIExpression::ApplyOffset, Offset);
1888 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
1889 } else if (MI.hasOneMemOperand()) {
1890 // Only describe memory which provably does not escape the function. As
1891 // described in llvm.org/PR43343, escaped memory may be clobbered by the
1892 // callee (or by another thread).
1893 const MachineFrameInfo &MFI = MF->getFrameInfo();
1894 const MachineMemOperand *MMO = MI.memoperands()[0];
1895 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1896
1897 // If the address points to "special" memory (e.g. a spill slot), it's
1898 // sufficient to check that it isn't aliased by any high-level IR value.
1899 if (!PSV || PSV->mayAlias(&MFI))
1900 return std::nullopt;
1901
1902 const MachineOperand *BaseOp;
1903 if (!getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, &TRI))
1904 return std::nullopt;
1905
1906 // FIXME: Scalable offsets are not yet handled in the offset code below.
1907 if (OffsetIsScalable)
1908 return std::nullopt;
1909
1910 // TODO: Can currently only handle mem instructions with a single define.
1911 // An example from the x86 target:
1912 // ...
1913 // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx
1914 // ...
1915 //
1916 if (MI.getNumExplicitDefs() != 1)
1917 return std::nullopt;
1918
1919 // TODO: In what way do we need to take Reg into consideration here?
1920
1921 SmallVector<uint64_t, 8> Ops;
1922 DIExpression::appendOffset(Ops, Offset);
1923 Ops.push_back(Elt: dwarf::DW_OP_deref_size);
1924 Ops.push_back(Elt: MMO->getSize().hasValue() ? MMO->getSize().getValue()
1925 : ~UINT64_C(0));
1926 Expr = DIExpression::prependOpcodes(Expr, Ops);
1927 return ParamLoadedValue(*BaseOp, Expr);
1928 }
1929
1930 return std::nullopt;
1931}
1932
1933// Get the call frame size just before MI.
1934unsigned TargetInstrInfo::getCallFrameSizeAt(MachineInstr &MI) const {
1935 // Search backwards from MI for the most recent call frame instruction.
1936 MachineBasicBlock *MBB = MI.getParent();
1937 for (auto &AdjI : reverse(C: make_range(x: MBB->instr_begin(), y: MI.getIterator()))) {
1938 if (AdjI.getOpcode() == getCallFrameSetupOpcode())
1939 return getFrameTotalSize(I: AdjI);
1940 if (AdjI.getOpcode() == getCallFrameDestroyOpcode())
1941 return 0;
1942 }
1943
1944 // If none was found, use the call frame size from the start of the basic
1945 // block.
1946 return MBB->getCallFrameSize();
1947}
1948
1949/// Both DefMI and UseMI must be valid. By default, call directly to the
1950/// itinerary. This may be overriden by the target.
1951std::optional<unsigned> TargetInstrInfo::getOperandLatency(
1952 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
1953 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
1954 unsigned DefClass = DefMI.getDesc().getSchedClass();
1955 unsigned UseClass = UseMI.getDesc().getSchedClass();
1956 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1957}
1958
1959bool TargetInstrInfo::getRegSequenceInputs(
1960 const MachineInstr &MI, unsigned DefIdx,
1961 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1962 assert((MI.isRegSequence() ||
1963 MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1964
1965 if (!MI.isRegSequence())
1966 return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1967
1968 // We are looking at:
1969 // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
1970 assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
1971 for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
1972 OpIdx += 2) {
1973 const MachineOperand &MOReg = MI.getOperand(i: OpIdx);
1974 if (MOReg.isUndef())
1975 continue;
1976 const MachineOperand &MOSubIdx = MI.getOperand(i: OpIdx + 1);
1977 assert(MOSubIdx.isImm() &&
1978 "One of the subindex of the reg_sequence is not an immediate");
1979 // Record Reg:SubReg, SubIdx.
1980 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
1981 (unsigned)MOSubIdx.getImm()));
1982 }
1983 return true;
1984}
1985
1986bool TargetInstrInfo::getExtractSubregInputs(
1987 const MachineInstr &MI, unsigned DefIdx,
1988 RegSubRegPairAndIdx &InputReg) const {
1989 assert((MI.isExtractSubreg() ||
1990 MI.isExtractSubregLike()) && "Instruction do not have the proper type");
1991
1992 if (!MI.isExtractSubreg())
1993 return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
1994
1995 // We are looking at:
1996 // Def = EXTRACT_SUBREG v0.sub1, sub0.
1997 assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
1998 const MachineOperand &MOReg = MI.getOperand(i: 1);
1999 if (MOReg.isUndef())
2000 return false;
2001 const MachineOperand &MOSubIdx = MI.getOperand(i: 2);
2002 assert(MOSubIdx.isImm() &&
2003 "The subindex of the extract_subreg is not an immediate");
2004
2005 InputReg.Reg = MOReg.getReg();
2006 InputReg.SubReg = MOReg.getSubReg();
2007 InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
2008 return true;
2009}
2010
2011bool TargetInstrInfo::getInsertSubregInputs(
2012 const MachineInstr &MI, unsigned DefIdx,
2013 RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
2014 assert((MI.isInsertSubreg() ||
2015 MI.isInsertSubregLike()) && "Instruction do not have the proper type");
2016
2017 if (!MI.isInsertSubreg())
2018 return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
2019
2020 // We are looking at:
2021 // Def = INSERT_SEQUENCE v0, v1, sub0.
2022 assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
2023 const MachineOperand &MOBaseReg = MI.getOperand(i: 1);
2024 const MachineOperand &MOInsertedReg = MI.getOperand(i: 2);
2025 if (MOInsertedReg.isUndef())
2026 return false;
2027 const MachineOperand &MOSubIdx = MI.getOperand(i: 3);
2028 assert(MOSubIdx.isImm() &&
2029 "One of the subindex of the reg_sequence is not an immediate");
2030 BaseReg.Reg = MOBaseReg.getReg();
2031 BaseReg.SubReg = MOBaseReg.getSubReg();
2032
2033 InsertedReg.Reg = MOInsertedReg.getReg();
2034 InsertedReg.SubReg = MOInsertedReg.getSubReg();
2035 InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
2036 return true;
2037}
2038
2039// Returns a MIRPrinter comment for this machine operand.
2040std::string TargetInstrInfo::createMIROperandComment(
2041 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2042 const TargetRegisterInfo * /*RemoveMe*/) const {
2043
2044 if (!MI.isInlineAsm())
2045 return "";
2046
2047 std::string Flags;
2048 raw_string_ostream OS(Flags);
2049
2050 if (OpIdx == InlineAsm::MIOp_ExtraInfo) {
2051 // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
2052 unsigned ExtraInfo = Op.getImm();
2053 OS << interleaved(R: InlineAsm::getExtraInfoNames(ExtraInfo), Separator: " ");
2054 return Flags;
2055 }
2056
2057 int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx);
2058 if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx)
2059 return "";
2060
2061 assert(Op.isImm() && "Expected flag operand to be an immediate");
2062 // Pretty print the inline asm operand descriptor.
2063 unsigned Flag = Op.getImm();
2064 const InlineAsm::Flag F(Flag);
2065 OS << F.getKindName();
2066
2067 unsigned RCID;
2068 if (!F.isImmKind() && !F.isMemKind() && F.hasRegClassConstraint(RC&: RCID))
2069 OS << ':' << TRI.getRegClassName(Class: TRI.getRegClass(i: RCID));
2070
2071 if (F.isMemKind()) {
2072 InlineAsm::ConstraintCode MCID = F.getMemoryConstraintID();
2073 OS << ":" << InlineAsm::getMemConstraintName(C: MCID);
2074 }
2075
2076 unsigned TiedTo;
2077 if (F.isUseOperandTiedToDef(Idx&: TiedTo))
2078 OS << " tiedto:$" << TiedTo;
2079
2080 if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isRegUseKind()) &&
2081 F.getRegMayBeFolded())
2082 OS << " foldable";
2083
2084 return Flags;
2085}
2086
2087TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() = default;
2088
2089void TargetInstrInfo::mergeOutliningCandidateAttributes(
2090 Function &F, std::vector<outliner::Candidate> &Candidates) const {
2091 // Include target features from an arbitrary candidate for the outlined
2092 // function. This makes sure the outlined function knows what kinds of
2093 // instructions are going into it. This is fine, since all parent functions
2094 // must necessarily support the instructions that are in the outlined region.
2095 outliner::Candidate &FirstCand = Candidates.front();
2096 const Function &ParentFn = FirstCand.getMF()->getFunction();
2097 if (ParentFn.hasFnAttribute(Kind: "target-features"))
2098 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-features"));
2099 if (ParentFn.hasFnAttribute(Kind: "target-cpu"))
2100 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-cpu"));
2101
2102 // Set nounwind, so we don't generate eh_frame.
2103 if (llvm::all_of(Range&: Candidates, P: [](const outliner::Candidate &C) {
2104 return C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
2105 }))
2106 F.addFnAttr(Kind: Attribute::NoUnwind);
2107}
2108
2109outliner::InstrType
2110TargetInstrInfo::getOutliningType(const MachineModuleInfo &MMI,
2111 MachineBasicBlock::iterator &MIT,
2112 unsigned Flags) const {
2113 MachineInstr &MI = *MIT;
2114
2115 // NOTE: MI.isMetaInstruction() will match CFI_INSTRUCTION, but some targets
2116 // have support for outlining those. Special-case that here.
2117 if (MI.isCFIInstruction())
2118 // Just go right to the target implementation.
2119 return getOutliningTypeImpl(MMI, MIT, Flags);
2120
2121 // Be conservative about inline assembly.
2122 if (MI.isInlineAsm())
2123 return outliner::InstrType::Illegal;
2124
2125 // Labels generally can't safely be outlined.
2126 if (MI.isLabel())
2127 return outliner::InstrType::Illegal;
2128
2129 // Don't let debug instructions impact analysis.
2130 if (MI.isDebugInstr())
2131 return outliner::InstrType::Invisible;
2132
2133 // Some other special cases.
2134 switch (MI.getOpcode()) {
2135 case TargetOpcode::IMPLICIT_DEF:
2136 case TargetOpcode::KILL:
2137 case TargetOpcode::LIFETIME_START:
2138 case TargetOpcode::LIFETIME_END:
2139 return outliner::InstrType::Invisible;
2140 default:
2141 break;
2142 }
2143
2144 // Is this a terminator for a basic block?
2145 if (MI.isTerminator()) {
2146 // If this is a branch to another block, we can't outline it.
2147 if (!MI.getParent()->succ_empty())
2148 return outliner::InstrType::Illegal;
2149
2150 // Don't outline if the branch is not unconditional.
2151 if (isPredicated(MI))
2152 return outliner::InstrType::Illegal;
2153 }
2154
2155 // Make sure none of the operands of this instruction do anything that
2156 // might break if they're moved outside their current function.
2157 // This includes MachineBasicBlock references, BlockAddressses,
2158 // Constant pool indices and jump table indices.
2159 //
2160 // A quick note on MO_TargetIndex:
2161 // This doesn't seem to be used in any of the architectures that the
2162 // MachineOutliner supports, but it was still filtered out in all of them.
2163 // There was one exception (RISC-V), but MO_TargetIndex also isn't used there.
2164 // As such, this check is removed both here and in the target-specific
2165 // implementations. Instead, we assert to make sure this doesn't
2166 // catch anyone off-guard somewhere down the line.
2167 for (const MachineOperand &MOP : MI.operands()) {
2168 // If you hit this assertion, please remove it and adjust
2169 // `getOutliningTypeImpl` for your target appropriately if necessary.
2170 // Adding the assertion back to other supported architectures
2171 // would be nice too :)
2172 assert(!MOP.isTargetIndex() && "This isn't used quite yet!");
2173
2174 // CFI instructions should already have been filtered out at this point.
2175 assert(!MOP.isCFIIndex() && "CFI instructions handled elsewhere!");
2176
2177 // PrologEpilogInserter should've already run at this point.
2178 assert(!MOP.isFI() && "FrameIndex instructions should be gone by now!");
2179
2180 if (MOP.isMBB() || MOP.isBlockAddress() || MOP.isCPI() || MOP.isJTI())
2181 return outliner::InstrType::Illegal;
2182 }
2183
2184 // If we don't know, delegate to the target-specific hook.
2185 return getOutliningTypeImpl(MMI, MIT, Flags);
2186}
2187
2188bool TargetInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
2189 unsigned &Flags) const {
2190 // Some instrumentations create special TargetOpcode at the start which
2191 // expands to special code sequences which must be present.
2192 auto First = MBB.getFirstNonDebugInstr();
2193 if (First == MBB.end())
2194 return true;
2195
2196 if (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
2197 First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)
2198 return false;
2199
2200 // Some instrumentations create special pseudo-instructions at or just before
2201 // the end that must be present.
2202 auto Last = MBB.getLastNonDebugInstr();
2203 if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET ||
2204 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2205 return false;
2206
2207 if (Last != First && Last->isReturn()) {
2208 --Last;
2209 if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT ||
2210 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2211 return false;
2212 }
2213 return true;
2214}
2215
2216bool TargetInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
2217 return MI->isCall() || MI->hasUnmodeledSideEffects() ||
2218 (MI->hasOrderedMemoryRef() && !MI->isDereferenceableInvariantLoad());
2219}
2220