1//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetInstrInfo.h"
14#include "llvm/ADT/SmallSet.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/Dwarf.h"
17#include "llvm/CodeGen/MachineCombinerPattern.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineMemOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/MachineScheduler.h"
23#include "llvm/CodeGen/MachineTraceMetrics.h"
24#include "llvm/CodeGen/PseudoSourceValue.h"
25#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
26#include "llvm/CodeGen/StackMaps.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/CodeGen/TargetLowering.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/CodeGen/TargetSchedule.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCInstrItineraries.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Support/ErrorHandling.h"
38#include "llvm/Support/InterleavedRange.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Target/TargetMachine.h"
41
42using namespace llvm;
43
44static cl::opt<bool> DisableHazardRecognizer(
45 "disable-sched-hazard", cl::Hidden, cl::init(Val: false),
46 cl::desc("Disable hazard detection during preRA scheduling"));
47
48static cl::opt<bool> EnableAccReassociation(
49 "acc-reassoc", cl::Hidden, cl::init(Val: true),
50 cl::desc("Enable reassociation of accumulation chains"));
51
52static cl::opt<unsigned int>
53 MinAccumulatorDepth("acc-min-depth", cl::Hidden, cl::init(Val: 8),
54 cl::desc("Minimum length of accumulator chains "
55 "required for the optimization to kick in"));
56
57static cl::opt<unsigned int> MaxAccumulatorWidth(
58 "acc-max-width", cl::Hidden, cl::init(Val: 3),
59 cl::desc("Maximum number of branches in the accumulator tree"));
60
61TargetInstrInfo::~TargetInstrInfo() = default;
62
63const TargetRegisterClass *TargetInstrInfo::getRegClass(const MCInstrDesc &MCID,
64 unsigned OpNum) const {
65 if (OpNum >= MCID.getNumOperands())
66 return nullptr;
67
68 const MCOperandInfo &OpInfo = MCID.operands()[OpNum];
69 int16_t RegClass = getOpRegClassID(OpInfo);
70
71 // Instructions like INSERT_SUBREG do not have fixed register classes.
72 if (RegClass < 0)
73 return nullptr;
74
75 // Otherwise just look it up normally.
76 return TRI.getRegClass(i: RegClass);
77}
78
79/// insertNoop - Insert a noop into the instruction stream at the specified
80/// point.
81void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB,
82 MachineBasicBlock::iterator MI) const {
83 llvm_unreachable("Target didn't implement insertNoop!");
84}
85
86/// insertNoops - Insert noops into the instruction stream at the specified
87/// point.
88void TargetInstrInfo::insertNoops(MachineBasicBlock &MBB,
89 MachineBasicBlock::iterator MI,
90 unsigned Quantity) const {
91 for (unsigned i = 0; i < Quantity; ++i)
92 insertNoop(MBB, MI);
93}
94
95static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
96 return strncmp(s1: Str, s2: MAI.getCommentString().data(),
97 n: MAI.getCommentString().size()) == 0;
98}
99
100/// Measure the specified inline asm to determine an approximation of its
101/// length.
102/// Comments (which run till the next SeparatorString or newline) do not
103/// count as an instruction.
104/// Any other non-whitespace text is considered an instruction, with
105/// multiple instructions separated by SeparatorString or newlines.
106/// Variable-length instructions are not handled here; this function
107/// may be overloaded in the target code to do that.
108/// We implement a special case of the .space directive which takes only a
109/// single integer argument in base 10 that is the size in bytes. This is a
110/// restricted form of the GAS directive in that we only interpret
111/// simple--i.e. not a logical or arithmetic expression--size values without
112/// the optional fill value. This is primarily used for creating arbitrary
113/// sized inline asm blocks for testing purposes.
114unsigned TargetInstrInfo::getInlineAsmLength(
115 const char *Str,
116 const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const {
117 // Count the number of instructions in the asm.
118 bool AtInsnStart = true;
119 unsigned Length = 0;
120 const unsigned MaxInstLength = MAI.getMaxInstLength(STI);
121 for (; *Str; ++Str) {
122 if (*Str == '\n' || strncmp(s1: Str, s2: MAI.getSeparatorString(),
123 n: strlen(s: MAI.getSeparatorString())) == 0) {
124 AtInsnStart = true;
125 } else if (isAsmComment(Str, MAI)) {
126 // Stop counting as an instruction after a comment until the next
127 // separator.
128 AtInsnStart = false;
129 }
130
131 if (AtInsnStart && !isSpace(C: static_cast<unsigned char>(*Str))) {
132 unsigned AddLength = MaxInstLength;
133 if (strncmp(s1: Str, s2: ".space", n: 6) == 0) {
134 char *EStr;
135 int SpaceSize;
136 SpaceSize = strtol(nptr: Str + 6, endptr: &EStr, base: 10);
137 SpaceSize = SpaceSize < 0 ? 0 : SpaceSize;
138 while (*EStr != '\n' && isSpace(C: static_cast<unsigned char>(*EStr)))
139 ++EStr;
140 if (*EStr == '\0' || *EStr == '\n' ||
141 isAsmComment(Str: EStr, MAI)) // Successfully parsed .space argument
142 AddLength = SpaceSize;
143 }
144 Length += AddLength;
145 AtInsnStart = false;
146 }
147 }
148
149 return Length;
150}
151
152unsigned TargetInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
153 unsigned Size = 0;
154 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
155 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
156 while (++I != E && I->isInsideBundle()) {
157 assert(!I->isBundle() && "No nested bundle!");
158 Size += getInstSizeInBytes(MI: *I);
159 }
160
161 return Size;
162}
163
164/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
165/// after it, replacing it with an unconditional branch to NewDest.
166void
167TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
168 MachineBasicBlock *NewDest) const {
169 MachineBasicBlock *MBB = Tail->getParent();
170
171 // Remove all the old successors of MBB from the CFG.
172 while (!MBB->succ_empty())
173 MBB->removeSuccessor(I: MBB->succ_begin());
174
175 // Save off the debug loc before erasing the instruction.
176 DebugLoc DL = Tail->getDebugLoc();
177
178 // Update call info and remove all the dead instructions
179 // from the end of MBB.
180 while (Tail != MBB->end()) {
181 auto MI = Tail++;
182 if (MI->shouldUpdateAdditionalCallInfo())
183 MBB->getParent()->eraseAdditionalCallInfo(MI: &*MI);
184 MBB->erase(I: MI);
185 }
186
187 // If MBB isn't immediately before MBB, insert a branch to it.
188 if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
189 insertBranch(MBB&: *MBB, TBB: NewDest, FBB: nullptr, Cond: SmallVector<MachineOperand, 0>(), DL);
190 MBB->addSuccessor(Succ: NewDest);
191}
192
193MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI,
194 bool NewMI, unsigned Idx1,
195 unsigned Idx2) const {
196 const MCInstrDesc &MCID = MI.getDesc();
197 bool HasDef = MCID.getNumDefs();
198 if (HasDef && !MI.getOperand(i: 0).isReg())
199 // No idea how to commute this instruction. Target should implement its own.
200 return nullptr;
201
202 unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
203 unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
204 assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
205 CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
206 "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
207 assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
208 "This only knows how to commute register operands so far");
209
210 Register Reg0 = HasDef ? MI.getOperand(i: 0).getReg() : Register();
211 Register Reg1 = MI.getOperand(i: Idx1).getReg();
212 Register Reg2 = MI.getOperand(i: Idx2).getReg();
213 unsigned SubReg0 = HasDef ? MI.getOperand(i: 0).getSubReg() : 0;
214 unsigned SubReg1 = MI.getOperand(i: Idx1).getSubReg();
215 unsigned SubReg2 = MI.getOperand(i: Idx2).getSubReg();
216 bool Reg1IsKill = MI.getOperand(i: Idx1).isKill();
217 bool Reg2IsKill = MI.getOperand(i: Idx2).isKill();
218 bool Reg1IsUndef = MI.getOperand(i: Idx1).isUndef();
219 bool Reg2IsUndef = MI.getOperand(i: Idx2).isUndef();
220 bool Reg1IsInternal = MI.getOperand(i: Idx1).isInternalRead();
221 bool Reg2IsInternal = MI.getOperand(i: Idx2).isInternalRead();
222 // Avoid calling isRenamable for virtual registers since we assert that
223 // renamable property is only queried/set for physical registers.
224 bool Reg1IsRenamable =
225 Reg1.isPhysical() ? MI.getOperand(i: Idx1).isRenamable() : false;
226 bool Reg2IsRenamable =
227 Reg2.isPhysical() ? MI.getOperand(i: Idx2).isRenamable() : false;
228
229 // For a case like this:
230 // %0.sub = INST %0.sub(tied), %1.sub, implicit-def %0
231 // we need to update the implicit-def after commuting to result in:
232 // %1.sub = INST %1.sub(tied), %0.sub, implicit-def %1
233 SmallVector<unsigned> UpdateImplicitDefIdx;
234 if (HasDef && MI.hasImplicitDef()) {
235 for (auto [OpNo, MO] : llvm::enumerate(First: MI.implicit_operands())) {
236 Register ImplReg = MO.getReg();
237 if ((ImplReg.isVirtual() && ImplReg == Reg0) ||
238 (ImplReg.isPhysical() && Reg0.isPhysical() &&
239 TRI.isSubRegisterEq(RegA: ImplReg, RegB: Reg0)))
240 UpdateImplicitDefIdx.push_back(Elt: OpNo + MI.getNumExplicitOperands());
241 }
242 }
243
244 // If destination is tied to either of the commuted source register, then
245 // it must be updated.
246 if (HasDef && Reg0 == Reg1 &&
247 MI.getDesc().getOperandConstraint(OpNum: Idx1, Constraint: MCOI::TIED_TO) == 0) {
248 Reg2IsKill = false;
249 Reg0 = Reg2;
250 SubReg0 = SubReg2;
251 } else if (HasDef && Reg0 == Reg2 &&
252 MI.getDesc().getOperandConstraint(OpNum: Idx2, Constraint: MCOI::TIED_TO) == 0) {
253 Reg1IsKill = false;
254 Reg0 = Reg1;
255 SubReg0 = SubReg1;
256 }
257
258 MachineInstr *CommutedMI = nullptr;
259 if (NewMI) {
260 // Create a new instruction.
261 MachineFunction &MF = *MI.getMF();
262 CommutedMI = MF.CloneMachineInstr(Orig: &MI);
263 } else {
264 CommutedMI = &MI;
265 }
266
267 if (HasDef) {
268 CommutedMI->getOperand(i: 0).setReg(Reg0);
269 CommutedMI->getOperand(i: 0).setSubReg(SubReg0);
270 for (unsigned Idx : UpdateImplicitDefIdx)
271 CommutedMI->getOperand(i: Idx).setReg(Reg0);
272 }
273 CommutedMI->getOperand(i: Idx2).setReg(Reg1);
274 CommutedMI->getOperand(i: Idx1).setReg(Reg2);
275 CommutedMI->getOperand(i: Idx2).setSubReg(SubReg1);
276 CommutedMI->getOperand(i: Idx1).setSubReg(SubReg2);
277 CommutedMI->getOperand(i: Idx2).setIsKill(Reg1IsKill);
278 CommutedMI->getOperand(i: Idx1).setIsKill(Reg2IsKill);
279 CommutedMI->getOperand(i: Idx2).setIsUndef(Reg1IsUndef);
280 CommutedMI->getOperand(i: Idx1).setIsUndef(Reg2IsUndef);
281 CommutedMI->getOperand(i: Idx2).setIsInternalRead(Reg1IsInternal);
282 CommutedMI->getOperand(i: Idx1).setIsInternalRead(Reg2IsInternal);
283 // Avoid calling setIsRenamable for virtual registers since we assert that
284 // renamable property is only queried/set for physical registers.
285 if (Reg1.isPhysical())
286 CommutedMI->getOperand(i: Idx2).setIsRenamable(Reg1IsRenamable);
287 if (Reg2.isPhysical())
288 CommutedMI->getOperand(i: Idx1).setIsRenamable(Reg2IsRenamable);
289 return CommutedMI;
290}
291
292MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI,
293 unsigned OpIdx1,
294 unsigned OpIdx2) const {
295 // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
296 // any commutable operand, which is done in findCommutedOpIndices() method
297 // called below.
298 if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) &&
299 !findCommutedOpIndices(MI, SrcOpIdx1&: OpIdx1, SrcOpIdx2&: OpIdx2)) {
300 assert(MI.isCommutable() &&
301 "Precondition violation: MI must be commutable.");
302 return nullptr;
303 }
304 return commuteInstructionImpl(MI, NewMI, Idx1: OpIdx1, Idx2: OpIdx2);
305}
306
307bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
308 unsigned &ResultIdx2,
309 unsigned CommutableOpIdx1,
310 unsigned CommutableOpIdx2) {
311 if (ResultIdx1 == CommuteAnyOperandIndex &&
312 ResultIdx2 == CommuteAnyOperandIndex) {
313 ResultIdx1 = CommutableOpIdx1;
314 ResultIdx2 = CommutableOpIdx2;
315 } else if (ResultIdx1 == CommuteAnyOperandIndex) {
316 if (ResultIdx2 == CommutableOpIdx1)
317 ResultIdx1 = CommutableOpIdx2;
318 else if (ResultIdx2 == CommutableOpIdx2)
319 ResultIdx1 = CommutableOpIdx1;
320 else
321 return false;
322 } else if (ResultIdx2 == CommuteAnyOperandIndex) {
323 if (ResultIdx1 == CommutableOpIdx1)
324 ResultIdx2 = CommutableOpIdx2;
325 else if (ResultIdx1 == CommutableOpIdx2)
326 ResultIdx2 = CommutableOpIdx1;
327 else
328 return false;
329 } else
330 // Check that the result operand indices match the given commutable
331 // operand indices.
332 return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) ||
333 (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1);
334
335 return true;
336}
337
338bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
339 unsigned &SrcOpIdx1,
340 unsigned &SrcOpIdx2) const {
341 assert(!MI.isBundle() &&
342 "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
343
344 const MCInstrDesc &MCID = MI.getDesc();
345 if (!MCID.isCommutable())
346 return false;
347
348 // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
349 // is not true, then the target must implement this.
350 unsigned CommutableOpIdx1 = MCID.getNumDefs();
351 unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
352 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2,
353 CommutableOpIdx1, CommutableOpIdx2))
354 return false;
355
356 if (!MI.getOperand(i: SrcOpIdx1).isReg() || !MI.getOperand(i: SrcOpIdx2).isReg())
357 // No idea.
358 return false;
359 return true;
360}
361
362bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
363 if (!MI.isTerminator()) return false;
364
365 // Conditional branch is a special case.
366 if (MI.isBranch() && !MI.isBarrier())
367 return true;
368 if (!MI.isPredicable())
369 return true;
370 return !isPredicated(MI);
371}
372
373bool TargetInstrInfo::PredicateInstruction(
374 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
375 bool MadeChange = false;
376
377 assert(!MI.isBundle() &&
378 "TargetInstrInfo::PredicateInstruction() can't handle bundles");
379
380 const MCInstrDesc &MCID = MI.getDesc();
381 if (!MI.isPredicable())
382 return false;
383
384 for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
385 if (MCID.operands()[i].isPredicate()) {
386 MachineOperand &MO = MI.getOperand(i);
387 if (MO.isReg()) {
388 MO.setReg(Pred[j].getReg());
389 MadeChange = true;
390 } else if (MO.isImm()) {
391 MO.setImm(Pred[j].getImm());
392 MadeChange = true;
393 } else if (MO.isMBB()) {
394 MO.setMBB(Pred[j].getMBB());
395 MadeChange = true;
396 }
397 ++j;
398 }
399 }
400 return MadeChange;
401}
402
403bool TargetInstrInfo::hasLoadFromStackSlot(
404 const MachineInstr &MI,
405 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
406 size_t StartSize = Accesses.size();
407 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
408 oe = MI.memoperands_end();
409 o != oe; ++o) {
410 if ((*o)->isLoad() &&
411 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
412 Accesses.push_back(Elt: *o);
413 }
414 return Accesses.size() != StartSize;
415}
416
417bool TargetInstrInfo::hasStoreToStackSlot(
418 const MachineInstr &MI,
419 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
420 size_t StartSize = Accesses.size();
421 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
422 oe = MI.memoperands_end();
423 o != oe; ++o) {
424 if ((*o)->isStore() &&
425 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
426 Accesses.push_back(Elt: *o);
427 }
428 return Accesses.size() != StartSize;
429}
430
431bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
432 unsigned SubIdx, unsigned &Size,
433 unsigned &Offset,
434 const MachineFunction &MF) const {
435 if (!SubIdx) {
436 Size = TRI.getSpillSize(RC: *RC);
437 Offset = 0;
438 return true;
439 }
440 unsigned BitSize = TRI.getSubRegIdxSize(Idx: SubIdx);
441 // Convert bit size to byte size.
442 if (BitSize % 8)
443 return false;
444
445 int BitOffset = TRI.getSubRegIdxOffset(Idx: SubIdx);
446 if (BitOffset < 0 || BitOffset % 8)
447 return false;
448
449 Size = BitSize / 8;
450 Offset = (unsigned)BitOffset / 8;
451
452 assert(TRI.getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
453
454 if (!MF.getDataLayout().isLittleEndian()) {
455 Offset = TRI.getSpillSize(RC: *RC) - (Offset + Size);
456 }
457 return true;
458}
459
460void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
461 MachineBasicBlock::iterator I,
462 Register DestReg, unsigned SubIdx,
463 const MachineInstr &Orig,
464 LaneBitmask UsedLanes) const {
465 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
466 MI->substituteRegister(FromReg: MI->getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
467 MBB.insert(I, MI);
468}
469
470bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0,
471 const MachineInstr &MI1,
472 const MachineRegisterInfo *MRI) const {
473 return MI0.isIdenticalTo(Other: MI1, Check: MachineInstr::IgnoreVRegDefs);
474}
475
476MachineInstr &
477TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
478 MachineBasicBlock::iterator InsertBefore,
479 const MachineInstr &Orig) const {
480 MachineFunction &MF = *MBB.getParent();
481 // CFI instructions are marked as non-duplicable, because Darwin compact
482 // unwind info emission can't handle multiple prologue setups.
483 assert((!Orig.isNotDuplicable() ||
484 (!MF.getFunction().getParent()->getTargetTriple().isOSDarwin() &&
485 Orig.isCFIInstruction())) &&
486 "Instruction cannot be duplicated");
487
488 return MF.cloneMachineInstrBundle(MBB, InsertBefore, Orig);
489}
490
491// If the COPY instruction in MI can be folded to a stack operation, return
492// the register class to use.
493static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI,
494 const TargetInstrInfo &TII,
495 unsigned FoldIdx) {
496 assert(TII.isCopyInstr(MI) && "MI must be a COPY instruction");
497 if (MI.getNumOperands() != 2)
498 return nullptr;
499 assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
500
501 const MachineOperand &FoldOp = MI.getOperand(i: FoldIdx);
502 const MachineOperand &LiveOp = MI.getOperand(i: 1 - FoldIdx);
503
504 if (FoldOp.getSubReg() || LiveOp.getSubReg())
505 return nullptr;
506
507 Register FoldReg = FoldOp.getReg();
508 Register LiveReg = LiveOp.getReg();
509
510 assert(FoldReg.isVirtual() && "Cannot fold physregs");
511
512 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
513 const TargetRegisterClass *RC = MRI.getRegClass(Reg: FoldReg);
514
515 if (LiveOp.getReg().isPhysical())
516 return RC->contains(Reg: LiveOp.getReg()) ? RC : nullptr;
517
518 if (RC->hasSubClassEq(RC: MRI.getRegClass(Reg: LiveReg)))
519 return RC;
520
521 // FIXME: Allow folding when register classes are memory compatible.
522 return nullptr;
523}
524
525MCInst TargetInstrInfo::getNop() const { llvm_unreachable("Not implemented"); }
526
527/// Try to remove the load by folding it to a register
528/// operand at the use. We fold the load instructions if load defines a virtual
529/// register, the virtual register is used once in the same BB, and the
530/// instructions in-between do not load or store, and have no side effects.
531MachineInstr *TargetInstrInfo::optimizeLoadInstr(MachineInstr &MI,
532 const MachineRegisterInfo *MRI,
533 Register &FoldAsLoadDefReg,
534 MachineInstr *&DefMI,
535 MachineInstr *&CopyMI) const {
536 // Check whether we can move DefMI here.
537 DefMI = MRI->getVRegDef(Reg: FoldAsLoadDefReg);
538 assert(DefMI);
539 bool SawStore = false;
540 if (!DefMI->isSafeToMove(SawStore))
541 return nullptr;
542
543 // Collect information about virtual register operands of MI.
544 SmallVector<unsigned, 1> SrcOperandIds;
545 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
546 MachineOperand &MO = MI.getOperand(i);
547 if (!MO.isReg())
548 continue;
549 Register Reg = MO.getReg();
550 if (Reg != FoldAsLoadDefReg)
551 continue;
552 // Do not fold if we have a subreg use or a def.
553 if (MO.getSubReg() || MO.isDef())
554 return nullptr;
555 SrcOperandIds.push_back(Elt: i);
556 }
557 if (SrcOperandIds.empty())
558 return nullptr;
559
560 // Check whether we can fold the def into SrcOperandId.
561 if (MachineInstr *FoldMI =
562 foldMemoryOperand(MI, Ops: SrcOperandIds, LoadMI&: *DefMI, CopyMI)) {
563 FoldAsLoadDefReg = 0;
564 return FoldMI;
565 }
566
567 return nullptr;
568}
569
570std::pair<unsigned, unsigned>
571TargetInstrInfo::getPatchpointUnfoldableRange(const MachineInstr &MI) const {
572 switch (MI.getOpcode()) {
573 case TargetOpcode::STACKMAP:
574 // StackMapLiveValues are foldable
575 return std::make_pair(x: 0, y: StackMapOpers(&MI).getVarIdx());
576 case TargetOpcode::PATCHPOINT:
577 // For PatchPoint, the call args are not foldable (even if reported in the
578 // stackmap e.g. via anyregcc).
579 return std::make_pair(x: 0, y: PatchPointOpers(&MI).getVarIdx());
580 case TargetOpcode::STATEPOINT:
581 // For statepoints, fold deopt and gc arguments, but not call arguments.
582 return std::make_pair(x: MI.getNumDefs(), y: StatepointOpers(&MI).getVarIdx());
583 default:
584 llvm_unreachable("unexpected stackmap opcode");
585 }
586}
587
588static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
589 ArrayRef<unsigned> Ops, int FrameIndex,
590 const TargetInstrInfo &TII) {
591 unsigned StartIdx = 0;
592 unsigned NumDefs = 0;
593 // getPatchpointUnfoldableRange throws guarantee if MI is not a patchpoint.
594 std::tie(args&: NumDefs, args&: StartIdx) = TII.getPatchpointUnfoldableRange(MI);
595
596 unsigned DefToFoldIdx = MI.getNumOperands();
597
598 // Return false if any operands requested for folding are not foldable (not
599 // part of the stackmap's live values).
600 for (unsigned Op : Ops) {
601 if (Op < NumDefs) {
602 assert(DefToFoldIdx == MI.getNumOperands() && "Folding multiple defs");
603 DefToFoldIdx = Op;
604 } else if (Op < StartIdx) {
605 return nullptr;
606 }
607 if (MI.getOperand(i: Op).isTied())
608 return nullptr;
609 }
610
611 MachineInstr *NewMI =
612 MF.CreateMachineInstr(MCID: TII.get(Opcode: MI.getOpcode()), DL: MI.getDebugLoc(), NoImplicit: true);
613 MachineInstrBuilder MIB(MF, NewMI);
614
615 // No need to fold return, the meta data, and function arguments
616 for (unsigned i = 0; i < StartIdx; ++i)
617 if (i != DefToFoldIdx)
618 MIB.add(MO: MI.getOperand(i));
619
620 for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) {
621 MachineOperand &MO = MI.getOperand(i);
622 unsigned TiedTo = e;
623 (void)MI.isRegTiedToDefOperand(UseOpIdx: i, DefOpIdx: &TiedTo);
624
625 if (is_contained(Range&: Ops, Element: i)) {
626 assert(TiedTo == e && "Cannot fold tied operands");
627 unsigned SpillSize;
628 unsigned SpillOffset;
629 // Compute the spill slot size and offset.
630 const TargetRegisterClass *RC =
631 MF.getRegInfo().getRegClass(Reg: MO.getReg());
632 bool Valid =
633 TII.getStackSlotRange(RC, SubIdx: MO.getSubReg(), Size&: SpillSize, Offset&: SpillOffset, MF);
634 if (!Valid)
635 report_fatal_error(reason: "cannot spill patchpoint subregister operand");
636 MIB.addImm(Val: StackMaps::IndirectMemRefOp);
637 MIB.addImm(Val: SpillSize);
638 MIB.addFrameIndex(Idx: FrameIndex);
639 MIB.addImm(Val: SpillOffset);
640 } else {
641 MIB.add(MO);
642 if (TiedTo < e) {
643 assert(TiedTo < NumDefs && "Bad tied operand");
644 if (TiedTo > DefToFoldIdx)
645 --TiedTo;
646 NewMI->tieOperands(DefIdx: TiedTo, UseIdx: NewMI->getNumOperands() - 1);
647 }
648 }
649 }
650 return NewMI;
651}
652
653static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI,
654 const TargetInstrInfo &TII) {
655 // If the machine operand is tied, untie it first.
656 if (MI->getOperand(i: OpNo).isTied()) {
657 unsigned TiedTo = MI->findTiedOperandIdx(OpIdx: OpNo);
658 MI->untieRegOperand(OpIdx: OpNo);
659 // Intentional recursion!
660 foldInlineAsmMemOperand(MI, OpNo: TiedTo, FI, TII);
661 }
662
663 SmallVector<MachineOperand, 5> NewOps;
664 TII.getFrameIndexOperands(Ops&: NewOps, FI);
665 assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands");
666 MI->removeOperand(OpNo);
667 MI->insert(InsertBefore: MI->operands_begin() + OpNo, Ops: NewOps);
668
669 // Change the previous operand to a MemKind InlineAsm::Flag. The second param
670 // is the per-target number of operands that represent the memory operand
671 // excluding this one (MD). This includes MO.
672 InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size());
673 F.setMemConstraint(InlineAsm::ConstraintCode::m);
674 MachineOperand &MD = MI->getOperand(i: OpNo - 1);
675 MD.setImm(F);
676}
677
678// Returns nullptr if not possible to fold.
679static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
680 ArrayRef<unsigned> Ops, int FI,
681 const TargetInstrInfo &TII) {
682 assert(MI.isInlineAsm() && "wrong opcode");
683 if (Ops.size() > 1)
684 return nullptr;
685 unsigned Op = Ops[0];
686 assert(Op && "should never be first operand");
687 assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands");
688
689 if (!MI.mayFoldInlineAsmRegOp(OpId: Op))
690 return nullptr;
691
692 MachineInstr &NewMI = TII.duplicate(MBB&: *MI.getParent(), InsertBefore: MI.getIterator(), Orig: MI);
693
694 foldInlineAsmMemOperand(MI: &NewMI, OpNo: Op, FI, TII);
695
696 // Update mayload/maystore metadata, and memoperands.
697 const VirtRegInfo &RI =
698 AnalyzeVirtRegInBundle(MI, Reg: MI.getOperand(i: Op).getReg());
699 MachineOperand &ExtraMO = NewMI.getOperand(i: InlineAsm::MIOp_ExtraInfo);
700 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
701 if (RI.Reads) {
702 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad);
703 Flags |= MachineMemOperand::MOLoad;
704 }
705 if (RI.Writes) {
706 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
707 Flags |= MachineMemOperand::MOStore;
708 }
709 MachineFunction *MF = NewMI.getMF();
710 const MachineFrameInfo &MFI = MF->getFrameInfo();
711 MachineMemOperand *MMO = MF->getMachineMemOperand(
712 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: Flags, Size: MFI.getObjectSize(ObjectIdx: FI),
713 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
714 NewMI.addMemOperand(MF&: *MF, MO: MMO);
715
716 return &NewMI;
717}
718
719MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
720 ArrayRef<unsigned> Ops, int FI,
721 MachineInstr *&CopyMI,
722 LiveIntervals *LIS,
723 VirtRegMap *VRM) const {
724 auto Flags = MachineMemOperand::MONone;
725 for (unsigned OpIdx : Ops)
726 Flags |= MI.getOperand(i: OpIdx).isDef() ? MachineMemOperand::MOStore
727 : MachineMemOperand::MOLoad;
728
729 MachineBasicBlock *MBB = MI.getParent();
730 assert(MBB && "foldMemoryOperand needs an inserted instruction");
731 MachineFunction &MF = *MBB->getParent();
732
733 // If we're not folding a load into a subreg, the size of the load is the
734 // size of the spill slot. But if we are, we need to figure out what the
735 // actual load size is.
736 int64_t MemSize = 0;
737 const MachineFrameInfo &MFI = MF.getFrameInfo();
738
739 if (Flags & MachineMemOperand::MOStore) {
740 MemSize = MFI.getObjectSize(ObjectIdx: FI);
741 } else {
742 for (unsigned OpIdx : Ops) {
743 int64_t OpSize = MFI.getObjectSize(ObjectIdx: FI);
744
745 if (auto SubReg = MI.getOperand(i: OpIdx).getSubReg()) {
746 unsigned SubRegSize = TRI.getSubRegIdxSize(Idx: SubReg);
747 if (SubRegSize > 0 && !(SubRegSize % 8))
748 OpSize = SubRegSize / 8;
749 }
750
751 MemSize = std::max(a: MemSize, b: OpSize);
752 }
753 }
754
755 assert(MemSize && "Did not expect a zero-sized stack slot");
756
757 MachineInstr *NewMI = nullptr;
758
759 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
760 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
761 MI.getOpcode() == TargetOpcode::STATEPOINT) {
762 // Fold stackmap/patchpoint.
763 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex: FI, TII: *this);
764 if (NewMI)
765 MBB->insert(I: MI, MI: NewMI);
766 } else if (MI.isInlineAsm()) {
767 return foldInlineAsmMemOperand(MI, Ops, FI, TII: *this);
768 } else {
769 // Ask the target to do the actual folding.
770 NewMI = foldMemoryOperandImpl(MF, MI, Ops, FrameIndex: FI, CopyMI, LIS, VRM);
771 }
772
773 if (NewMI) {
774 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
775 // Add a memory operand, foldMemoryOperandImpl doesn't do that.
776 assert((!(Flags & MachineMemOperand::MOStore) ||
777 NewMI->mayStore()) &&
778 "Folded a def to a non-store!");
779 assert((!(Flags & MachineMemOperand::MOLoad) ||
780 NewMI->mayLoad()) &&
781 "Folded a use to a non-load!");
782 assert(MFI.getObjectOffset(FI) != -1);
783 MachineMemOperand *MMO =
784 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF, FI),
785 F: Flags, Size: MemSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
786 NewMI->addMemOperand(MF, MO: MMO);
787
788 // The pass "x86 speculative load hardening" always attaches symbols to
789 // call instructions. We need copy it form old instruction.
790 NewMI->cloneInstrSymbols(MF, MI);
791
792 return NewMI;
793 }
794
795 // Straight COPY may fold as load/store.
796 if (!isCopyInstr(MI) || Ops.size() != 1)
797 return nullptr;
798
799 const TargetRegisterClass *RC = canFoldCopy(MI, TII: *this, FoldIdx: Ops[0]);
800 if (!RC)
801 return nullptr;
802
803 const MachineOperand &MO = MI.getOperand(i: 1 - Ops[0]);
804 MachineBasicBlock::iterator Pos = MI;
805 if (Flags == MachineMemOperand::MOStore) {
806 if (MO.isUndef()) {
807 // If this is an undef copy, we do not need to bother we inserting spill
808 // code.
809 BuildMI(BB&: *MBB, I: Pos, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::KILL)).add(MO);
810 } else {
811 storeRegToStackSlot(MBB&: *MBB, MI: Pos, SrcReg: MO.getReg(), isKill: MO.isKill(), FrameIndex: FI, RC,
812 VReg: Register());
813 }
814 } else
815 loadRegFromStackSlot(MBB&: *MBB, MI: Pos, DestReg: MO.getReg(), FrameIndex: FI, RC, VReg: Register());
816
817 return &*--Pos;
818}
819
820MachineInstr *
821TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
822 MachineInstr &LoadMI, MachineInstr *&CopyMI,
823 LiveIntervals *LIS, VirtRegMap *VRM) const {
824 assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
825#ifndef NDEBUG
826 for (unsigned OpIdx : Ops)
827 assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
828#endif
829
830 MachineBasicBlock &MBB = *MI.getParent();
831 MachineFunction &MF = *MBB.getParent();
832
833 // Ask the target to do the actual folding.
834 MachineInstr *NewMI = nullptr;
835 int FrameIndex = 0;
836
837 if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
838 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
839 MI.getOpcode() == TargetOpcode::STATEPOINT) &&
840 isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
841 // Fold stackmap/patchpoint.
842 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, TII: *this);
843 if (NewMI)
844 NewMI = &*MBB.insert(I: MI, MI: NewMI);
845 } else if (MI.isInlineAsm() && isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
846 return foldInlineAsmMemOperand(MI, Ops, FI: FrameIndex, TII: *this);
847 } else {
848 // Ask the target to do the actual folding.
849 NewMI = foldMemoryOperandImpl(MF, MI, Ops, LoadMI, CopyMI, LIS, VRM);
850 }
851
852 if (!NewMI)
853 return nullptr;
854
855 // Copy the memoperands from the load to the folded instruction.
856 if (MI.memoperands_empty()) {
857 NewMI->setMemRefs(MF, MemRefs: LoadMI.memoperands());
858 } else {
859 // Handle the rare case of folding multiple loads.
860 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
861 for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(),
862 E = LoadMI.memoperands_end();
863 I != E; ++I) {
864 NewMI->addMemOperand(MF, MO: *I);
865 }
866 }
867 return NewMI;
868}
869
870/// transferImplicitOperands - MI is a pseudo-instruction, and the lowered
871/// replacement instructions immediately precede it. Copy any implicit
872/// operands from MI to the replacement instruction.
873static void transferImplicitOperands(MachineInstr *MI,
874 const TargetRegisterInfo *TRI) {
875 MachineBasicBlock::iterator CopyMI = MI;
876 --CopyMI;
877
878 Register DstReg = MI->getOperand(i: 0).getReg();
879 for (const MachineOperand &MO : MI->implicit_operands()) {
880 CopyMI->addOperand(Op: MO);
881
882 // Be conservative about preserving kills when subregister defs are
883 // involved. If there was implicit kill of a super-register overlapping the
884 // copy result, we would kill the subregisters previous copies defined.
885
886 if (MO.isKill() && TRI->regsOverlap(RegA: DstReg, RegB: MO.getReg()))
887 CopyMI->getOperand(i: CopyMI->getNumOperands() - 1).setIsKill(false);
888 }
889}
890
891void TargetInstrInfo::lowerCopy(MachineInstr *MI) const {
892 if (MI->allDefsAreDead()) {
893 MI->setDesc(get(Opcode: TargetOpcode::KILL));
894 return;
895 }
896
897 MachineOperand &DstMO = MI->getOperand(i: 0);
898 MachineOperand &SrcMO = MI->getOperand(i: 1);
899
900 bool IdentityCopy = (SrcMO.getReg() == DstMO.getReg());
901 if (IdentityCopy || SrcMO.isUndef()) {
902 // No need to insert an identity copy instruction, but replace with a KILL
903 // if liveness is changed.
904 if (SrcMO.isUndef() || MI->getNumOperands() > 2) {
905 // We must make sure the super-register gets killed. Replace the
906 // instruction with KILL.
907 MI->setDesc(get(Opcode: TargetOpcode::KILL));
908 return;
909 }
910 // Vanilla identity copy.
911 MI->eraseFromParent();
912 return;
913 }
914
915 copyPhysReg(MBB&: *MI->getParent(), MI, DL: MI->getDebugLoc(), DestReg: DstMO.getReg(),
916 SrcReg: SrcMO.getReg(), KillSrc: SrcMO.isKill(),
917 RenamableDest: DstMO.getReg().isPhysical() ? DstMO.isRenamable() : false,
918 RenamableSrc: SrcMO.getReg().isPhysical() ? SrcMO.isRenamable() : false);
919
920 if (MI->getNumOperands() > 2)
921 transferImplicitOperands(MI, TRI: &TRI);
922 MI->eraseFromParent();
923}
924
925bool TargetInstrInfo::hasReassociableOperands(
926 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
927 const MachineOperand &Op1 = Inst.getOperand(i: 1);
928 const MachineOperand &Op2 = Inst.getOperand(i: 2);
929 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
930
931 // We need virtual register definitions for the operands that we will
932 // reassociate.
933 MachineInstr *MI1 = nullptr;
934 MachineInstr *MI2 = nullptr;
935 if (Op1.isReg() && Op1.getReg().isVirtual())
936 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
937 if (Op2.isReg() && Op2.getReg().isVirtual())
938 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
939
940 // And at least one operand must be defined in MBB.
941 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
942}
943
944bool TargetInstrInfo::areOpcodesEqualOrInverse(unsigned Opcode1,
945 unsigned Opcode2) const {
946 return Opcode1 == Opcode2 || getInverseOpcode(Opcode: Opcode1) == Opcode2;
947}
948
949bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
950 bool &Commuted) const {
951 const MachineBasicBlock *MBB = Inst.getParent();
952 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
953 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 1).getReg());
954 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
955 unsigned Opcode = Inst.getOpcode();
956
957 // If only one operand has the same or inverse opcode and it's the second
958 // source operand, the operands must be commuted.
959 Commuted = !areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
960 areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI2->getOpcode());
961 if (Commuted)
962 std::swap(a&: MI1, b&: MI2);
963
964 // 1. The previous instruction must be the same type as Inst.
965 // 2. The previous instruction must also be associative/commutative or be the
966 // inverse of such an operation (this can be different even for
967 // instructions with the same opcode if traits like fast-math-flags are
968 // included).
969 // 3. The previous instruction must have virtual register definitions for its
970 // operands in the same basic block as Inst.
971 // 4. The previous instruction's result must only be used by Inst.
972 return areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
973 (isAssociativeAndCommutative(Inst: *MI1) ||
974 isAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
975 hasReassociableOperands(Inst: *MI1, MBB) &&
976 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
977}
978
979// 1. The operation must be associative and commutative or be the inverse of
980// such an operation.
981// 2. The instruction must have virtual register definitions for its
982// operands in the same basic block.
983// 3. The instruction must have a reassociable sibling.
984bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
985 bool &Commuted) const {
986 return (isAssociativeAndCommutative(Inst) ||
987 isAssociativeAndCommutative(Inst, /* Invert */ true)) &&
988 hasReassociableOperands(Inst, MBB: Inst.getParent()) &&
989 hasReassociableSibling(Inst, Commuted);
990}
991
992// Utility routine that checks if \param MO is defined by an
993// \param CombineOpc instruction in the basic block \param MBB.
994// If \param CombineOpc is not provided, the OpCode check will
995// be skipped.
996static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
997 unsigned CombineOpc = 0) {
998 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
999 MachineInstr *MI = nullptr;
1000
1001 if (MO.isReg() && MO.getReg().isVirtual())
1002 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
1003 // And it needs to be in the trace (otherwise, it won't have a depth).
1004 if (!MI || MI->getParent() != &MBB ||
1005 (MI->getOpcode() != CombineOpc && CombineOpc != 0))
1006 return false;
1007 // Must only used by the user we combine with.
1008 if (!MRI.hasOneNonDBGUse(RegNo: MO.getReg()))
1009 return false;
1010
1011 return true;
1012}
1013
1014// A chain of accumulation instructions will be selected IFF:
1015// 1. All the accumulation instructions in the chain have the same opcode,
1016// besides the first that has a slightly different opcode because it does
1017// not accumulate into a register.
1018// 2. All the instructions in the chain are combinable (have a single use
1019// which itself is part of the chain).
1020// 3. Meets the required minimum length.
1021void TargetInstrInfo::getAccumulatorChain(
1022 MachineInstr *CurrentInstr, SmallVectorImpl<Register> &Chain) const {
1023 // Walk up the chain of accumulation instructions and collect them in the
1024 // vector.
1025 MachineBasicBlock &MBB = *CurrentInstr->getParent();
1026 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1027 unsigned AccumulatorOpcode = CurrentInstr->getOpcode();
1028 std::optional<unsigned> ChainStartOpCode =
1029 getAccumulationStartOpcode(Opcode: AccumulatorOpcode);
1030
1031 if (!ChainStartOpCode.has_value())
1032 return;
1033
1034 // Push the first accumulator result to the start of the chain.
1035 Chain.push_back(Elt: CurrentInstr->getOperand(i: 0).getReg());
1036
1037 // Collect the accumulator input register from all instructions in the chain.
1038 while (CurrentInstr &&
1039 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1), CombineOpc: AccumulatorOpcode)) {
1040 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1041 CurrentInstr = MRI.getUniqueVRegDef(Reg: CurrentInstr->getOperand(i: 1).getReg());
1042 }
1043
1044 // Add the instruction at the top of the chain.
1045 if (CurrentInstr->getOpcode() == AccumulatorOpcode &&
1046 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1)))
1047 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1048}
1049
1050/// Find chains of accumulations that can be rewritten as a tree for increased
1051/// ILP.
1052bool TargetInstrInfo::getAccumulatorReassociationPatterns(
1053 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns) const {
1054 if (!EnableAccReassociation)
1055 return false;
1056
1057 unsigned Opc = Root.getOpcode();
1058 if (!isAccumulationOpcode(Opcode: Opc))
1059 return false;
1060
1061 // Verify that this is the end of the chain.
1062 MachineBasicBlock &MBB = *Root.getParent();
1063 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1064 if (!MRI.hasOneNonDBGUser(RegNo: Root.getOperand(i: 0).getReg()))
1065 return false;
1066
1067 auto User = MRI.use_instr_nodbg_begin(RegNo: Root.getOperand(i: 0).getReg());
1068 if (User->getOpcode() == Opc)
1069 return false;
1070
1071 // Walk up the use chain and collect the reduction chain.
1072 SmallVector<Register, 32> Chain;
1073 getAccumulatorChain(CurrentInstr: &Root, Chain);
1074
1075 // Reject chains which are too short to be worth modifying.
1076 if (Chain.size() < MinAccumulatorDepth)
1077 return false;
1078
1079 // Check if the MBB this instruction is a part of contains any other chains.
1080 // If so, don't apply it.
1081 SmallSet<Register, 32> ReductionChain(llvm::from_range, Chain);
1082 for (const auto &I : MBB) {
1083 if (I.getOpcode() == Opc &&
1084 !ReductionChain.contains(V: I.getOperand(i: 0).getReg()))
1085 return false;
1086 }
1087
1088 Patterns.push_back(Elt: MachineCombinerPattern::ACC_CHAIN);
1089 return true;
1090}
1091
1092// Reduce branches of the accumulator tree by adding them together.
1093void TargetInstrInfo::reduceAccumulatorTree(
1094 SmallVectorImpl<Register> &RegistersToReduce,
1095 SmallVectorImpl<MachineInstr *> &InsInstrs, MachineFunction &MF,
1096 MachineInstr &Root, MachineRegisterInfo &MRI,
1097 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
1098 Register ResultReg) const {
1099 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1100 SmallVector<Register, 8> NewRegs;
1101
1102 // Get the opcode for the reduction instruction we will need to build.
1103 // If for some reason it is not defined, early exit and don't apply this.
1104 unsigned ReduceOpCode = getReduceOpcodeForAccumulator(AccumulatorOpCode: Root.getOpcode());
1105
1106 for (unsigned int i = 1; i <= (RegistersToReduce.size() / 2); i += 2) {
1107 auto RHS = RegistersToReduce[i - 1];
1108 auto LHS = RegistersToReduce[i];
1109 Register Dest;
1110 // If we are reducing 2 registers, reuse the original result register.
1111 if (RegistersToReduce.size() == 2)
1112 Dest = ResultReg;
1113 // Otherwise, create a new virtual register to hold the partial sum.
1114 else {
1115 auto NewVR = MRI.createVirtualRegister(
1116 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()));
1117 Dest = NewVR;
1118 NewRegs.push_back(Elt: Dest);
1119 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: Dest, y: InsInstrs.size()));
1120 }
1121
1122 // Create the new reduction instruction.
1123 MachineInstrBuilder MIB =
1124 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: ReduceOpCode), DestReg: Dest)
1125 .addReg(RegNo: RHS)
1126 .addReg(RegNo: LHS);
1127 // Copy any flags needed from the original instruction.
1128 MIB->setFlags(Root.getFlags());
1129 InsInstrs.push_back(Elt: MIB);
1130 }
1131
1132 // If the number of registers to reduce is odd, add the remaining register to
1133 // the vector of registers to reduce.
1134 if (RegistersToReduce.size() % 2 != 0)
1135 NewRegs.push_back(Elt: RegistersToReduce[RegistersToReduce.size() - 1]);
1136
1137 RegistersToReduce = std::move(NewRegs);
1138}
1139
1140// The concept of the reassociation pass is that these operations can benefit
1141// from this kind of transformation:
1142//
1143// A = ? op ?
1144// B = A op X (Prev)
1145// C = B op Y (Root)
1146// -->
1147// A = ? op ?
1148// B = X op Y
1149// C = A op B
1150//
1151// breaking the dependency between A and B, allowing them to be executed in
1152// parallel (or back-to-back in a pipeline) instead of depending on each other.
1153
1154// FIXME: This has the potential to be expensive (compile time) while not
1155// improving the code at all. Some ways to limit the overhead:
1156// 1. Track successful transforms; bail out if hit rate gets too low.
1157// 2. Only enable at -O3 or some other non-default optimization level.
1158// 3. Pre-screen pattern candidates here: if an operand of the previous
1159// instruction is known to not increase the critical path, then don't match
1160// that pattern.
1161bool TargetInstrInfo::getMachineCombinerPatterns(
1162 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
1163 bool DoRegPressureReduce) const {
1164 bool Commute;
1165 if (isReassociationCandidate(Inst: Root, Commuted&: Commute)) {
1166 // We found a sequence of instructions that may be suitable for a
1167 // reassociation of operands to increase ILP. Specify each commutation
1168 // possibility for the Prev instruction in the sequence and let the
1169 // machine combiner decide if changing the operands is worthwhile.
1170 if (Commute) {
1171 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_YB);
1172 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_YB);
1173 } else {
1174 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_BY);
1175 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_BY);
1176 }
1177 return true;
1178 }
1179 if (getAccumulatorReassociationPatterns(Root, Patterns))
1180 return true;
1181
1182 return false;
1183}
1184
1185/// Return true when a code sequence can improve loop throughput.
1186bool TargetInstrInfo::isThroughputPattern(unsigned Pattern) const {
1187 return false;
1188}
1189
1190CombinerObjective
1191TargetInstrInfo::getCombinerObjective(unsigned Pattern) const {
1192 switch (Pattern) {
1193 case MachineCombinerPattern::ACC_CHAIN:
1194 return CombinerObjective::MustReduceDepth;
1195 default:
1196 return CombinerObjective::Default;
1197 }
1198}
1199
1200std::pair<unsigned, unsigned>
1201TargetInstrInfo::getReassociationOpcodes(unsigned Pattern,
1202 const MachineInstr &Root,
1203 const MachineInstr &Prev) const {
1204 bool AssocCommutRoot = isAssociativeAndCommutative(Inst: Root);
1205 bool AssocCommutPrev = isAssociativeAndCommutative(Inst: Prev);
1206
1207 // Early exit if both opcodes are associative and commutative. It's a trivial
1208 // reassociation when we only change operands order. In this case opcodes are
1209 // not required to have inverse versions.
1210 if (AssocCommutRoot && AssocCommutPrev) {
1211 assert(Root.getOpcode() == Prev.getOpcode() && "Expected to be equal");
1212 return std::make_pair(x: Root.getOpcode(), y: Root.getOpcode());
1213 }
1214
1215 // At least one instruction is not associative or commutative.
1216 // Since we have matched one of the reassociation patterns, we expect that the
1217 // instructions' opcodes are equal or one of them is the inversion of the
1218 // other.
1219 assert(areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()) &&
1220 "Incorrectly matched pattern");
1221 unsigned AssocCommutOpcode = Root.getOpcode();
1222 unsigned InverseOpcode = *getInverseOpcode(Opcode: Root.getOpcode());
1223 if (!AssocCommutRoot)
1224 std::swap(a&: AssocCommutOpcode, b&: InverseOpcode);
1225
1226 // The transformation rule (`+` is any associative and commutative binary
1227 // operation, `-` is the inverse):
1228 // REASSOC_AX_BY:
1229 // (A + X) + Y => A + (X + Y)
1230 // (A + X) - Y => A + (X - Y)
1231 // (A - X) + Y => A - (X - Y)
1232 // (A - X) - Y => A - (X + Y)
1233 // REASSOC_XA_BY:
1234 // (X + A) + Y => (X + Y) + A
1235 // (X + A) - Y => (X - Y) + A
1236 // (X - A) + Y => (X + Y) - A
1237 // (X - A) - Y => (X - Y) - A
1238 // REASSOC_AX_YB:
1239 // Y + (A + X) => (Y + X) + A
1240 // Y - (A + X) => (Y - X) - A
1241 // Y + (A - X) => (Y - X) + A
1242 // Y - (A - X) => (Y + X) - A
1243 // REASSOC_XA_YB:
1244 // Y + (X + A) => (Y + X) + A
1245 // Y - (X + A) => (Y - X) - A
1246 // Y + (X - A) => (Y + X) - A
1247 // Y - (X - A) => (Y - X) + A
1248 switch (Pattern) {
1249 default:
1250 llvm_unreachable("Unexpected pattern");
1251 case MachineCombinerPattern::REASSOC_AX_BY:
1252 if (!AssocCommutRoot && AssocCommutPrev)
1253 return {AssocCommutOpcode, InverseOpcode};
1254 if (AssocCommutRoot && !AssocCommutPrev)
1255 return {InverseOpcode, InverseOpcode};
1256 if (!AssocCommutRoot && !AssocCommutPrev)
1257 return {InverseOpcode, AssocCommutOpcode};
1258 break;
1259 case MachineCombinerPattern::REASSOC_XA_BY:
1260 if (!AssocCommutRoot && AssocCommutPrev)
1261 return {AssocCommutOpcode, InverseOpcode};
1262 if (AssocCommutRoot && !AssocCommutPrev)
1263 return {InverseOpcode, AssocCommutOpcode};
1264 if (!AssocCommutRoot && !AssocCommutPrev)
1265 return {InverseOpcode, InverseOpcode};
1266 break;
1267 case MachineCombinerPattern::REASSOC_AX_YB:
1268 if (!AssocCommutRoot && AssocCommutPrev)
1269 return {InverseOpcode, InverseOpcode};
1270 if (AssocCommutRoot && !AssocCommutPrev)
1271 return {AssocCommutOpcode, InverseOpcode};
1272 if (!AssocCommutRoot && !AssocCommutPrev)
1273 return {InverseOpcode, AssocCommutOpcode};
1274 break;
1275 case MachineCombinerPattern::REASSOC_XA_YB:
1276 if (!AssocCommutRoot && AssocCommutPrev)
1277 return {InverseOpcode, InverseOpcode};
1278 if (AssocCommutRoot && !AssocCommutPrev)
1279 return {InverseOpcode, AssocCommutOpcode};
1280 if (!AssocCommutRoot && !AssocCommutPrev)
1281 return {AssocCommutOpcode, InverseOpcode};
1282 break;
1283 }
1284 llvm_unreachable("Unhandled combination");
1285}
1286
1287// Return a pair of boolean flags showing if the new root and new prev operands
1288// must be swapped. See visual example of the rule in
1289// TargetInstrInfo::getReassociationOpcodes.
1290static std::pair<bool, bool> mustSwapOperands(unsigned Pattern) {
1291 switch (Pattern) {
1292 default:
1293 llvm_unreachable("Unexpected pattern");
1294 case MachineCombinerPattern::REASSOC_AX_BY:
1295 return {false, false};
1296 case MachineCombinerPattern::REASSOC_XA_BY:
1297 return {true, false};
1298 case MachineCombinerPattern::REASSOC_AX_YB:
1299 return {true, true};
1300 case MachineCombinerPattern::REASSOC_XA_YB:
1301 return {true, true};
1302 }
1303}
1304
1305void TargetInstrInfo::getReassociateOperandIndices(
1306 const MachineInstr &Root, unsigned Pattern,
1307 std::array<unsigned, 5> &OperandIndices) const {
1308 switch (Pattern) {
1309 case MachineCombinerPattern::REASSOC_AX_BY:
1310 OperandIndices = {1, 1, 1, 2, 2};
1311 break;
1312 case MachineCombinerPattern::REASSOC_AX_YB:
1313 OperandIndices = {2, 1, 2, 2, 1};
1314 break;
1315 case MachineCombinerPattern::REASSOC_XA_BY:
1316 OperandIndices = {1, 2, 1, 1, 2};
1317 break;
1318 case MachineCombinerPattern::REASSOC_XA_YB:
1319 OperandIndices = {2, 2, 2, 1, 1};
1320 break;
1321 default:
1322 llvm_unreachable("unexpected MachineCombinerPattern");
1323 }
1324}
1325
1326/// Attempt the reassociation transformation to reduce critical path length.
1327/// See the above comments before getMachineCombinerPatterns().
1328void TargetInstrInfo::reassociateOps(
1329 MachineInstr &Root, MachineInstr &Prev, unsigned Pattern,
1330 SmallVectorImpl<MachineInstr *> &InsInstrs,
1331 SmallVectorImpl<MachineInstr *> &DelInstrs,
1332 ArrayRef<unsigned> OperandIndices,
1333 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
1334 MachineFunction *MF = Root.getMF();
1335 MachineRegisterInfo &MRI = MF->getRegInfo();
1336 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1337 const TargetRegisterClass *RC = Root.getRegClassConstraint(OpIdx: 0, TII, TRI: &TRI);
1338
1339 MachineOperand &OpA = Prev.getOperand(i: OperandIndices[1]);
1340 MachineOperand &OpB = Root.getOperand(i: OperandIndices[2]);
1341 MachineOperand &OpX = Prev.getOperand(i: OperandIndices[3]);
1342 MachineOperand &OpY = Root.getOperand(i: OperandIndices[4]);
1343 MachineOperand &OpC = Root.getOperand(i: 0);
1344
1345 Register RegA = OpA.getReg();
1346 unsigned SubRegA = OpA.getSubReg();
1347 Register RegB = OpB.getReg();
1348 Register RegX = OpX.getReg();
1349 unsigned SubRegX = OpX.getSubReg();
1350 Register RegY = OpY.getReg();
1351 unsigned SubRegY = OpY.getSubReg();
1352 Register RegC = OpC.getReg();
1353
1354 if (RegA.isVirtual())
1355 MRI.constrainRegClass(Reg: RegA, RC);
1356 if (RegB.isVirtual())
1357 MRI.constrainRegClass(Reg: RegB, RC);
1358 if (RegX.isVirtual())
1359 MRI.constrainRegClass(Reg: RegX, RC);
1360 if (RegY.isVirtual())
1361 MRI.constrainRegClass(Reg: RegY, RC);
1362 if (RegC.isVirtual())
1363 MRI.constrainRegClass(Reg: RegC, RC);
1364
1365 // Create a new virtual register for the result of (X op Y) instead of
1366 // recycling RegB because the MachineCombiner's computation of the critical
1367 // path requires a new register definition rather than an existing one.
1368 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
1369 unsigned SubRegNewVR = 0;
1370 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
1371
1372 auto [NewRootOpc, NewPrevOpc] = getReassociationOpcodes(Pattern, Root, Prev);
1373
1374 auto [SwapRootOperands, SwapPrevOperands] = mustSwapOperands(Pattern);
1375
1376 if (SwapPrevOperands) {
1377 std::swap(a&: RegX, b&: RegY);
1378 std::swap(a&: SubRegX, b&: SubRegY);
1379 }
1380
1381 unsigned PrevFirstOpIdx, PrevSecondOpIdx;
1382 unsigned RootFirstOpIdx, RootSecondOpIdx;
1383 switch (Pattern) {
1384 case MachineCombinerPattern::REASSOC_AX_BY:
1385 PrevFirstOpIdx = OperandIndices[1];
1386 PrevSecondOpIdx = OperandIndices[3];
1387 RootFirstOpIdx = OperandIndices[2];
1388 RootSecondOpIdx = OperandIndices[4];
1389 break;
1390 case MachineCombinerPattern::REASSOC_AX_YB:
1391 PrevFirstOpIdx = OperandIndices[1];
1392 PrevSecondOpIdx = OperandIndices[3];
1393 RootFirstOpIdx = OperandIndices[4];
1394 RootSecondOpIdx = OperandIndices[2];
1395 break;
1396 case MachineCombinerPattern::REASSOC_XA_BY:
1397 PrevFirstOpIdx = OperandIndices[3];
1398 PrevSecondOpIdx = OperandIndices[1];
1399 RootFirstOpIdx = OperandIndices[2];
1400 RootSecondOpIdx = OperandIndices[4];
1401 break;
1402 case MachineCombinerPattern::REASSOC_XA_YB:
1403 PrevFirstOpIdx = OperandIndices[3];
1404 PrevSecondOpIdx = OperandIndices[1];
1405 RootFirstOpIdx = OperandIndices[4];
1406 RootSecondOpIdx = OperandIndices[2];
1407 break;
1408 default:
1409 llvm_unreachable("unexpected MachineCombinerPattern");
1410 }
1411
1412 // Basically BuildMI but doesn't add implicit operands by default.
1413 auto buildMINoImplicit = [](MachineFunction &MF, const MIMetadata &MIMD,
1414 const MCInstrDesc &MCID, Register DestReg) {
1415 return MachineInstrBuilder(
1416 MF, MF.CreateMachineInstr(MCID, DL: MIMD.getDL(), /*NoImpl=*/NoImplicit: true))
1417 .copyMIMetadata(MIMD)
1418 .addReg(RegNo: DestReg, Flags: RegState::Define);
1419 };
1420
1421 // Create new instructions for insertion.
1422 MachineInstrBuilder MIB1 =
1423 buildMINoImplicit(*MF, MIMetadata(Prev), TII->get(Opcode: NewPrevOpc), NewVR);
1424 for (const auto &MO : Prev.explicit_operands()) {
1425 unsigned Idx = MO.getOperandNo();
1426 // Skip the result operand we'd already added.
1427 if (Idx == 0)
1428 continue;
1429 if (Idx == PrevFirstOpIdx)
1430 MIB1.addReg(RegNo: RegX, Flags: {}, SubReg: SubRegX);
1431 else if (Idx == PrevSecondOpIdx)
1432 MIB1.addReg(RegNo: RegY, Flags: {}, SubReg: SubRegY);
1433 else
1434 MIB1.add(MO);
1435 }
1436 MIB1.copyImplicitOps(OtherMI: Prev);
1437
1438 if (SwapRootOperands) {
1439 std::swap(a&: RegA, b&: NewVR);
1440 std::swap(a&: SubRegA, b&: SubRegNewVR);
1441 }
1442
1443 MachineInstrBuilder MIB2 =
1444 buildMINoImplicit(*MF, MIMetadata(Root), TII->get(Opcode: NewRootOpc), RegC);
1445 for (const auto &MO : Root.explicit_operands()) {
1446 unsigned Idx = MO.getOperandNo();
1447 // Skip the result operand.
1448 if (Idx == 0)
1449 continue;
1450 if (Idx == RootFirstOpIdx)
1451 MIB2 = MIB2.addReg(RegNo: RegA, Flags: {}, SubReg: SubRegA);
1452 else if (Idx == RootSecondOpIdx)
1453 MIB2 = MIB2.addReg(RegNo: NewVR, Flags: {}, SubReg: SubRegNewVR);
1454 else
1455 MIB2 = MIB2.add(MO);
1456 }
1457 MIB2.copyImplicitOps(OtherMI: Root);
1458
1459 // Propagate FP flags from the original instructions.
1460 // But clear poison-generating flags because those may not be valid now.
1461 // TODO: There should be a helper function for copying only fast-math-flags.
1462 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
1463 MIB1->setFlags(IntersectedFlags);
1464 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1465 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1466 MIB1->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1467 MIB1->clearFlag(Flag: MachineInstr::MIFlag::Disjoint);
1468
1469 MIB2->setFlags(IntersectedFlags);
1470 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1471 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1472 MIB2->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1473 MIB2->clearFlag(Flag: MachineInstr::MIFlag::Disjoint);
1474
1475 setSpecialOperandAttr(OldMI1&: Root, OldMI2&: Prev, NewMI1&: *MIB1, NewMI2&: *MIB2);
1476
1477 // Record new instructions for insertion and old instructions for deletion.
1478 InsInstrs.push_back(Elt: MIB1);
1479 InsInstrs.push_back(Elt: MIB2);
1480 DelInstrs.push_back(Elt: &Prev);
1481 DelInstrs.push_back(Elt: &Root);
1482
1483 // We transformed:
1484 // B = A op X (Prev)
1485 // C = B op Y (Root)
1486 // Into:
1487 // B = X op Y (MIB1)
1488 // C = A op B (MIB2)
1489 // C has the same value as before, B doesn't; as such, keep the debug number
1490 // of C but not of B.
1491 if (unsigned OldRootNum = Root.peekDebugInstrNum())
1492 MIB2.getInstr()->setDebugInstrNum(OldRootNum);
1493}
1494
1495void TargetInstrInfo::genAlternativeCodeSequence(
1496 MachineInstr &Root, unsigned Pattern,
1497 SmallVectorImpl<MachineInstr *> &InsInstrs,
1498 SmallVectorImpl<MachineInstr *> &DelInstrs,
1499 DenseMap<Register, unsigned> &InstIdxForVirtReg) const {
1500 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1501 MachineBasicBlock &MBB = *Root.getParent();
1502 MachineFunction &MF = *MBB.getParent();
1503 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1504
1505 switch (Pattern) {
1506 case MachineCombinerPattern::REASSOC_AX_BY:
1507 case MachineCombinerPattern::REASSOC_AX_YB:
1508 case MachineCombinerPattern::REASSOC_XA_BY:
1509 case MachineCombinerPattern::REASSOC_XA_YB: {
1510 // Select the previous instruction in the sequence based on the input
1511 // pattern.
1512 std::array<unsigned, 5> OperandIndices;
1513 getReassociateOperandIndices(Root, Pattern, OperandIndices);
1514 MachineInstr *Prev =
1515 MRI.getUniqueVRegDef(Reg: Root.getOperand(i: OperandIndices[0]).getReg());
1516
1517 // Don't reassociate if Prev and Root are in different blocks.
1518 if (Prev->getParent() != Root.getParent())
1519 return;
1520
1521 reassociateOps(Root, Prev&: *Prev, Pattern, InsInstrs, DelInstrs, OperandIndices,
1522 InstrIdxForVirtReg&: InstIdxForVirtReg);
1523 break;
1524 }
1525 case MachineCombinerPattern::ACC_CHAIN: {
1526 SmallVector<Register, 32> ChainRegs;
1527 getAccumulatorChain(CurrentInstr: &Root, Chain&: ChainRegs);
1528 unsigned int Depth = ChainRegs.size();
1529 assert(MaxAccumulatorWidth > 1 &&
1530 "Max accumulator width set to illegal value");
1531 unsigned int MaxWidth = Log2_32(Value: Depth) < MaxAccumulatorWidth
1532 ? Log2_32(Value: Depth)
1533 : MaxAccumulatorWidth;
1534
1535 // Walk down the chain and rewrite it as a tree.
1536 for (auto IndexedReg : llvm::enumerate(First: llvm::reverse(C&: ChainRegs))) {
1537 // No need to rewrite the first node, it is already perfect as it is.
1538 if (IndexedReg.index() == 0)
1539 continue;
1540
1541 // FIXME: Losing subregisters
1542 MachineInstr *Instr = MRI.getUniqueVRegDef(Reg: IndexedReg.value());
1543 MachineInstrBuilder MIB;
1544 Register AccReg;
1545 if (IndexedReg.index() < MaxWidth) {
1546 // Now we need to create new instructions for the first row.
1547 AccReg = Instr->getOperand(i: 0).getReg();
1548 unsigned OpCode = getAccumulationStartOpcode(Opcode: Root.getOpcode());
1549
1550 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: OpCode), DestReg: AccReg)
1551 .addReg(RegNo: Instr->getOperand(i: 2).getReg())
1552 .addReg(RegNo: Instr->getOperand(i: 3).getReg());
1553 } else {
1554 // For the remaining cases, we need to use an output register of one of
1555 // the newly inserted instructions as operand 1
1556 AccReg = Instr->getOperand(i: 0).getReg() == Root.getOperand(i: 0).getReg()
1557 ? MRI.createVirtualRegister(
1558 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()))
1559 : Instr->getOperand(i: 0).getReg();
1560 assert(IndexedReg.index() >= MaxWidth);
1561 auto AccumulatorInput =
1562 ChainRegs[Depth - (IndexedReg.index() - MaxWidth) - 1];
1563 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: Instr->getOpcode()),
1564 DestReg: AccReg)
1565 .addReg(RegNo: AccumulatorInput)
1566 .addReg(RegNo: Instr->getOperand(i: 2).getReg())
1567 .addReg(RegNo: Instr->getOperand(i: 3).getReg());
1568 }
1569
1570 MIB->setFlags(Instr->getFlags());
1571 InstIdxForVirtReg.insert(KV: std::make_pair(x&: AccReg, y: InsInstrs.size()));
1572 InsInstrs.push_back(Elt: MIB);
1573 DelInstrs.push_back(Elt: Instr);
1574 }
1575
1576 SmallVector<Register, 8> RegistersToReduce;
1577 for (unsigned i = (InsInstrs.size() - MaxWidth); i < InsInstrs.size();
1578 ++i) {
1579 auto Reg = InsInstrs[i]->getOperand(i: 0).getReg();
1580 RegistersToReduce.push_back(Elt: Reg);
1581 }
1582
1583 while (RegistersToReduce.size() > 1)
1584 reduceAccumulatorTree(RegistersToReduce, InsInstrs, MF, Root, MRI,
1585 InstrIdxForVirtReg&: InstIdxForVirtReg, ResultReg: Root.getOperand(i: 0).getReg());
1586
1587 break;
1588 }
1589 }
1590}
1591
1592MachineTraceStrategy TargetInstrInfo::getMachineCombinerTraceStrategy() const {
1593 return MachineTraceStrategy::TS_MinInstrCount;
1594}
1595
1596bool TargetInstrInfo::isReMaterializableImpl(
1597 const MachineInstr &MI) const {
1598 const MachineFunction &MF = *MI.getMF();
1599 const MachineRegisterInfo &MRI = MF.getRegInfo();
1600
1601 // Remat clients assume operand 0 is the defined register.
1602 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
1603 return false;
1604 Register DefReg = MI.getOperand(i: 0).getReg();
1605
1606 // A sub-register definition can only be rematerialized if the instruction
1607 // doesn't read the other parts of the register. Otherwise it is really a
1608 // read-modify-write operation on the full virtual register which cannot be
1609 // moved safely.
1610 if (DefReg.isVirtual() && MI.getOperand(i: 0).getSubReg() &&
1611 MI.readsVirtualRegister(Reg: DefReg))
1612 return false;
1613
1614 // A load from a fixed stack slot can be rematerialized. This may be
1615 // redundant with subsequent checks, but it's target-independent,
1616 // simple, and a common case.
1617 int FrameIdx = 0;
1618 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
1619 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
1620 return true;
1621
1622 // Avoid instructions obviously unsafe for remat.
1623 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
1624 MI.hasUnmodeledSideEffects())
1625 return false;
1626
1627 // Don't remat inline asm. We have no idea how expensive it is
1628 // even if it's side effect free.
1629 if (MI.isInlineAsm())
1630 return false;
1631
1632 // Avoid instructions which load from potentially varying memory.
1633 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
1634 return false;
1635
1636 // If any of the registers accessed are non-constant, conservatively assume
1637 // the instruction is not rematerializable.
1638 for (const MachineOperand &MO : MI.operands()) {
1639 if (!MO.isReg()) continue;
1640 Register Reg = MO.getReg();
1641 if (Reg == 0)
1642 continue;
1643
1644 // Check for a well-behaved physical register.
1645 if (Reg.isPhysical()) {
1646 if (MO.isUse()) {
1647 // If the physreg has no defs anywhere, it's just an ambient register
1648 // and we can freely move its uses. Alternatively, if it's allocatable,
1649 // it could get allocated to something with a def during allocation.
1650 if (!MRI.isConstantPhysReg(PhysReg: Reg))
1651 return false;
1652 } else {
1653 // A physreg def. We can't remat it.
1654 return false;
1655 }
1656 continue;
1657 }
1658
1659 // Only allow one virtual-register def. There may be multiple defs of the
1660 // same virtual register, though.
1661 if (MO.isDef() && Reg != DefReg)
1662 return false;
1663 }
1664
1665 // Everything checked out.
1666 return true;
1667}
1668
1669int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
1670 const MachineFunction *MF = MI.getMF();
1671 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
1672 bool StackGrowsDown =
1673 TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
1674
1675 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
1676 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
1677
1678 if (!isFrameInstr(I: MI))
1679 return 0;
1680
1681 int SPAdj = TFI->alignSPAdjust(SPAdj: getFrameSize(I: MI));
1682
1683 if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
1684 (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
1685 SPAdj = -SPAdj;
1686
1687 return SPAdj;
1688}
1689
1690/// isSchedulingBoundary - Test if the given instruction should be
1691/// considered a scheduling boundary. This primarily includes labels
1692/// and terminators.
1693bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1694 const MachineBasicBlock *MBB,
1695 const MachineFunction &MF) const {
1696 // Terminators and labels can't be scheduled around.
1697 if (MI.isTerminator() || MI.isPosition())
1698 return true;
1699
1700 // INLINEASM_BR can jump to another block
1701 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1702 return true;
1703
1704 // Don't attempt to schedule around any instruction that defines
1705 // a stack-oriented pointer, as it's unlikely to be profitable. This
1706 // saves compile time, because it doesn't require every single
1707 // stack slot reference to depend on the instruction that does the
1708 // modification.
1709 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
1710 return MI.modifiesRegister(Reg: TLI.getStackPointerRegisterToSaveRestore(), TRI: &TRI);
1711}
1712
1713// Provide a global flag for disabling the PreRA hazard recognizer that targets
1714// may choose to honor.
1715bool TargetInstrInfo::usePreRAHazardRecognizer() const {
1716 return !DisableHazardRecognizer;
1717}
1718
1719// Default implementation of CreateTargetRAHazardRecognizer.
1720ScheduleHazardRecognizer *TargetInstrInfo::
1721CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
1722 const ScheduleDAG *DAG) const {
1723 // Dummy hazard recognizer allows all instructions to issue.
1724 return new ScheduleHazardRecognizer();
1725}
1726
1727// Default implementation of CreateTargetMIHazardRecognizer.
1728ScheduleHazardRecognizer *TargetInstrInfo::CreateTargetMIHazardRecognizer(
1729 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
1730 return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler");
1731}
1732
1733// Default implementation of CreateTargetPostRAHazardRecognizer.
1734ScheduleHazardRecognizer *TargetInstrInfo::
1735CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1736 const ScheduleDAG *DAG) const {
1737 return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1738}
1739
1740// Default implementation of getMemOperandWithOffset.
1741bool TargetInstrInfo::getMemOperandWithOffset(const MachineInstr &MI,
1742 const MachineOperand *&BaseOp,
1743 int64_t &Offset,
1744 bool &OffsetIsScalable) const {
1745 SmallVector<const MachineOperand *, 4> BaseOps;
1746 LocationSize Width = LocationSize::precise(Value: 0);
1747 if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
1748 Width) ||
1749 BaseOps.size() != 1)
1750 return false;
1751 BaseOp = BaseOps.front();
1752 return true;
1753}
1754
1755//===----------------------------------------------------------------------===//
1756// SelectionDAG latency interface.
1757//===----------------------------------------------------------------------===//
1758
1759std::optional<unsigned>
1760TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1761 SDNode *DefNode, unsigned DefIdx,
1762 SDNode *UseNode, unsigned UseIdx) const {
1763 if (!ItinData || ItinData->isEmpty())
1764 return std::nullopt;
1765
1766 if (!DefNode->isMachineOpcode())
1767 return std::nullopt;
1768
1769 unsigned DefClass = get(Opcode: DefNode->getMachineOpcode()).getSchedClass();
1770 if (!UseNode->isMachineOpcode())
1771 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1772 unsigned UseClass = get(Opcode: UseNode->getMachineOpcode()).getSchedClass();
1773 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1774}
1775
1776unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1777 SDNode *N) const {
1778 if (!ItinData || ItinData->isEmpty())
1779 return 1;
1780
1781 if (!N->isMachineOpcode())
1782 return 1;
1783
1784 return ItinData->getStageLatency(ItinClassIndx: get(Opcode: N->getMachineOpcode()).getSchedClass());
1785}
1786
1787//===----------------------------------------------------------------------===//
1788// MachineInstr latency interface.
1789//===----------------------------------------------------------------------===//
1790
1791unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
1792 const MachineInstr &MI) const {
1793 if (!ItinData || ItinData->isEmpty())
1794 return 1;
1795
1796 unsigned Class = MI.getDesc().getSchedClass();
1797 int UOps = ItinData->Itineraries[Class].NumMicroOps;
1798 if (UOps >= 0)
1799 return UOps;
1800
1801 // The # of u-ops is dynamically determined. The specific target should
1802 // override this function to return the right number.
1803 return 1;
1804}
1805
1806/// Return the default expected latency for a def based on it's opcode.
1807unsigned TargetInstrInfo::defaultDefLatency(const TargetSubtargetInfo &STI,
1808 const MCSchedModel &SchedModel,
1809 const MachineInstr &DefMI) const {
1810 if (DefMI.isTransient())
1811 return 0;
1812 if (DefMI.mayLoad())
1813 return STI.getLoadLatency();
1814 if (isHighLatencyDef(opc: DefMI.getOpcode()))
1815 return SchedModel.HighLatency;
1816 return 1;
1817}
1818
1819unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const {
1820 return 0;
1821}
1822
1823unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1824 const MachineInstr &MI,
1825 unsigned *PredCost) const {
1826 // Default to one cycle for no itinerary. However, an "empty" itinerary may
1827 // still have a MinLatency property, which getStageLatency checks.
1828 if (!ItinData)
1829 return MI.mayLoad() ? 2 : 1;
1830
1831 return ItinData->getStageLatency(ItinClassIndx: MI.getDesc().getSchedClass());
1832}
1833
1834bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
1835 const MachineInstr &DefMI,
1836 unsigned DefIdx) const {
1837 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1838 if (!ItinData || ItinData->isEmpty())
1839 return false;
1840
1841 unsigned DefClass = DefMI.getDesc().getSchedClass();
1842 std::optional<unsigned> DefCycle =
1843 ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1844 return DefCycle && DefCycle <= 1U;
1845}
1846
1847bool TargetInstrInfo::isFunctionSafeToOutlineFrom(
1848 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
1849 const Function &F = MF.getFunction();
1850
1851 // Can F be deduplicated by the linker? If it can, don't outline from it.
1852 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
1853 return false;
1854
1855 // Allow outlining from named section functions if the target can place the
1856 // outlined function in the same section.
1857 if (F.hasSection() && !supportsSectionAwareOutlining())
1858 return false;
1859
1860 // It's safe to outline from MF.
1861 return true;
1862}
1863
1864bool TargetInstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
1865 // TODO: We don't split functions where a section attribute has been set
1866 // since the split part may not be placed in a contiguous region. It may also
1867 // be more beneficial to augment the linker to ensure contiguous layout of
1868 // split functions within the same section as specified by the attribute.
1869 if (MF.getFunction().hasSection())
1870 return false;
1871
1872 // We don't want to proceed further for cold functions
1873 // or functions of unknown hotness. Lukewarm functions have no prefix.
1874 std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
1875 if (SectionPrefix &&
1876 (*SectionPrefix == "unlikely" || *SectionPrefix == "unknown")) {
1877 return false;
1878 }
1879
1880 return true;
1881}
1882
1883std::optional<ParamLoadedValue>
1884TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
1885 Register Reg) const {
1886 const MachineFunction *MF = MI.getMF();
1887 DIExpression *Expr = DIExpression::get(Context&: MF->getFunction().getContext(), Elements: {});
1888 int64_t Offset;
1889 bool OffsetIsScalable;
1890
1891 // To simplify the sub-register handling, verify that we only need to
1892 // consider physical registers.
1893 assert(MF->getProperties().hasNoVRegs());
1894
1895 if (auto DestSrc = isCopyInstr(MI)) {
1896 Register DestReg = DestSrc->Destination->getReg();
1897
1898 // If the copy destination is the forwarding reg, describe the forwarding
1899 // reg using the copy source as the backup location. Example:
1900 //
1901 // x0 = MOV x7
1902 // call callee(x0) ; x0 described as x7
1903 if (Reg == DestReg)
1904 return ParamLoadedValue(*DestSrc->Source, Expr);
1905
1906 // If the target's hook couldn't describe this copy, give up.
1907 return std::nullopt;
1908 } else if (auto RegImm = isAddImmediate(MI, Reg)) {
1909 Register SrcReg = RegImm->Reg;
1910 Offset = RegImm->Imm;
1911 Expr = DIExpression::prepend(Expr, Flags: DIExpression::ApplyOffset, Offset);
1912 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
1913 } else if (MI.hasOneMemOperand()) {
1914 // Only describe memory which provably does not escape the function. As
1915 // described in llvm.org/PR43343, escaped memory may be clobbered by the
1916 // callee (or by another thread).
1917 const MachineFrameInfo &MFI = MF->getFrameInfo();
1918 const MachineMemOperand *MMO = MI.memoperands()[0];
1919 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1920
1921 // If the address points to "special" memory (e.g. a spill slot), it's
1922 // sufficient to check that it isn't aliased by any high-level IR value.
1923 if (!PSV || PSV->mayAlias(&MFI))
1924 return std::nullopt;
1925
1926 const MachineOperand *BaseOp;
1927 if (!getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable))
1928 return std::nullopt;
1929
1930 // FIXME: Scalable offsets are not yet handled in the offset code below.
1931 if (OffsetIsScalable)
1932 return std::nullopt;
1933
1934 // TODO: Can currently only handle mem instructions with a single define.
1935 // An example from the x86 target:
1936 // ...
1937 // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx
1938 // ...
1939 //
1940 if (MI.getNumExplicitDefs() != 1)
1941 return std::nullopt;
1942
1943 // DW_OP_deref_size needs a known, fixed size that fits in its one-byte
1944 // operand and is no larger than the size of an address.
1945 if (!MMO->getSize().hasValue() || MMO->getSize().isScalable() ||
1946 !isUInt<8>(x: MMO->getSize().getValue().getFixedValue()) ||
1947 MMO->getSize().getValue().getFixedValue() >
1948 MF->getDataLayout().getPointerSize())
1949 return std::nullopt;
1950
1951 // TODO: In what way do we need to take Reg into consideration here?
1952
1953 SmallVector<uint64_t, 8> Ops;
1954 DIExpression::appendOffset(Ops, Offset);
1955 Ops.push_back(Elt: dwarf::DW_OP_deref_size);
1956 Ops.push_back(Elt: MMO->getSize().getValue().getFixedValue());
1957 Expr = DIExpression::prependOpcodes(Expr, Ops);
1958 return ParamLoadedValue(*BaseOp, Expr);
1959 }
1960
1961 return std::nullopt;
1962}
1963
1964// Get the call frame size just before MI.
1965unsigned TargetInstrInfo::getCallFrameSizeAt(MachineInstr &MI) const {
1966 // Search backwards from MI for the most recent call frame instruction.
1967 MachineBasicBlock *MBB = MI.getParent();
1968 for (auto &AdjI : reverse(C: make_range(x: MBB->instr_begin(), y: MI.getIterator()))) {
1969 if (AdjI.getOpcode() == getCallFrameSetupOpcode())
1970 return getFrameTotalSize(I: AdjI);
1971 if (AdjI.getOpcode() == getCallFrameDestroyOpcode())
1972 return 0;
1973 }
1974
1975 // If none was found, use the call frame size from the start of the basic
1976 // block.
1977 return MBB->getCallFrameSize();
1978}
1979
1980/// Both DefMI and UseMI must be valid. By default, call directly to the
1981/// itinerary. This may be overriden by the target.
1982std::optional<unsigned> TargetInstrInfo::getOperandLatency(
1983 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
1984 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
1985 unsigned DefClass = DefMI.getDesc().getSchedClass();
1986 unsigned UseClass = UseMI.getDesc().getSchedClass();
1987 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1988}
1989
1990bool TargetInstrInfo::getRegSequenceInputs(
1991 const MachineInstr &MI, unsigned DefIdx,
1992 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1993 assert((MI.isRegSequence() ||
1994 MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1995
1996 if (!MI.isRegSequence())
1997 return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1998
1999 // We are looking at:
2000 // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
2001 assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
2002 for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
2003 OpIdx += 2) {
2004 const MachineOperand &MOReg = MI.getOperand(i: OpIdx);
2005 if (MOReg.isUndef())
2006 continue;
2007 const MachineOperand &MOSubIdx = MI.getOperand(i: OpIdx + 1);
2008 assert(MOSubIdx.isImm() &&
2009 "One of the subindex of the reg_sequence is not an immediate");
2010 // Record Reg:SubReg, SubIdx.
2011 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
2012 (unsigned)MOSubIdx.getImm()));
2013 }
2014 return true;
2015}
2016
2017bool TargetInstrInfo::getExtractSubregInputs(
2018 const MachineInstr &MI, unsigned DefIdx,
2019 RegSubRegPairAndIdx &InputReg) const {
2020 assert((MI.isExtractSubreg() ||
2021 MI.isExtractSubregLike()) && "Instruction do not have the proper type");
2022
2023 if (!MI.isExtractSubreg())
2024 return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
2025
2026 // We are looking at:
2027 // Def = EXTRACT_SUBREG v0.sub1, sub0.
2028 assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
2029 const MachineOperand &MOReg = MI.getOperand(i: 1);
2030 if (MOReg.isUndef())
2031 return false;
2032 const MachineOperand &MOSubIdx = MI.getOperand(i: 2);
2033 assert(MOSubIdx.isImm() &&
2034 "The subindex of the extract_subreg is not an immediate");
2035
2036 InputReg.Reg = MOReg.getReg();
2037 InputReg.SubReg = MOReg.getSubReg();
2038 InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
2039 return true;
2040}
2041
2042bool TargetInstrInfo::getInsertSubregInputs(
2043 const MachineInstr &MI, unsigned DefIdx,
2044 RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
2045 assert((MI.isInsertSubreg() ||
2046 MI.isInsertSubregLike()) && "Instruction do not have the proper type");
2047
2048 if (!MI.isInsertSubreg())
2049 return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
2050
2051 // We are looking at:
2052 // Def = INSERT_SEQUENCE v0, v1, sub0.
2053 assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
2054 const MachineOperand &MOBaseReg = MI.getOperand(i: 1);
2055 const MachineOperand &MOInsertedReg = MI.getOperand(i: 2);
2056 if (MOInsertedReg.isUndef())
2057 return false;
2058 const MachineOperand &MOSubIdx = MI.getOperand(i: 3);
2059 assert(MOSubIdx.isImm() &&
2060 "One of the subindex of the reg_sequence is not an immediate");
2061 BaseReg.Reg = MOBaseReg.getReg();
2062 BaseReg.SubReg = MOBaseReg.getSubReg();
2063
2064 InsertedReg.Reg = MOInsertedReg.getReg();
2065 InsertedReg.SubReg = MOInsertedReg.getSubReg();
2066 InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
2067 return true;
2068}
2069
2070// Returns a MIRPrinter comment for this machine operand.
2071std::string TargetInstrInfo::createMIROperandComment(const MachineInstr &MI,
2072 const MachineOperand &Op,
2073 unsigned OpIdx) const {
2074
2075 if (!MI.isInlineAsm())
2076 return "";
2077
2078 std::string Flags;
2079 raw_string_ostream OS(Flags);
2080
2081 if (OpIdx == InlineAsm::MIOp_ExtraInfo) {
2082 // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
2083 unsigned ExtraInfo = Op.getImm();
2084 OS << interleaved(R: InlineAsm::getExtraInfoNames(ExtraInfo), Separator: " ");
2085 return Flags;
2086 }
2087
2088 int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx);
2089 if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx)
2090 return "";
2091
2092 assert(Op.isImm() && "Expected flag operand to be an immediate");
2093 // Pretty print the inline asm operand descriptor.
2094 unsigned Flag = Op.getImm();
2095 const InlineAsm::Flag F(Flag);
2096 OS << F.getKindName();
2097
2098 unsigned RCID;
2099 if (!F.isImmKind() && !F.isMemKind() && F.hasRegClassConstraint(RC&: RCID))
2100 OS << ':' << TRI.getRegClassName(Class: TRI.getRegClass(i: RCID));
2101
2102 if (F.isMemKind()) {
2103 InlineAsm::ConstraintCode MCID = F.getMemoryConstraintID();
2104 OS << ":" << InlineAsm::getMemConstraintName(C: MCID);
2105 }
2106
2107 unsigned TiedTo;
2108 if (F.isUseOperandTiedToDef(Idx&: TiedTo))
2109 OS << " tiedto:$" << TiedTo;
2110
2111 if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isRegUseKind()) &&
2112 F.getRegMayBeFolded())
2113 OS << " foldable";
2114
2115 return Flags;
2116}
2117
2118TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() = default;
2119
2120void TargetInstrInfo::mergeOutliningCandidateAttributes(
2121 Function &F, std::vector<outliner::Candidate> &Candidates) const {
2122 // Include target features from an arbitrary candidate for the outlined
2123 // function. This makes sure the outlined function knows what kinds of
2124 // instructions are going into it. This is fine, since all parent functions
2125 // must necessarily support the instructions that are in the outlined region.
2126 outliner::Candidate &FirstCand = Candidates.front();
2127 const Function &ParentFn = FirstCand.getMF()->getFunction();
2128 if (ParentFn.hasFnAttribute(Kind: "target-features"))
2129 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-features"));
2130 if (ParentFn.hasFnAttribute(Kind: "target-cpu"))
2131 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-cpu"));
2132
2133 // Set nounwind, so we don't generate eh_frame.
2134 if (llvm::all_of(Range&: Candidates, P: [](const outliner::Candidate &C) {
2135 return C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
2136 }))
2137 F.addFnAttr(Kind: Attribute::NoUnwind);
2138}
2139
2140outliner::InstrType
2141TargetInstrInfo::getOutliningType(const MachineModuleInfo &MMI,
2142 MachineBasicBlock::iterator &MIT,
2143 unsigned Flags) const {
2144 MachineInstr &MI = *MIT;
2145
2146 // NOTE: MI.isMetaInstruction() will match CFI_INSTRUCTION, but some targets
2147 // have support for outlining those. Special-case that here.
2148 if (MI.isCFIInstruction())
2149 // Just go right to the target implementation.
2150 return getOutliningTypeImpl(MMI, MIT, Flags);
2151
2152 // Be conservative about inline assembly.
2153 if (MI.isInlineAsm())
2154 return outliner::InstrType::Illegal;
2155
2156 // Labels generally can't safely be outlined.
2157 if (MI.isLabel())
2158 return outliner::InstrType::Illegal;
2159
2160 // Don't let debug instructions impact analysis.
2161 if (MI.isDebugInstr())
2162 return outliner::InstrType::Invisible;
2163
2164 // Some other special cases.
2165 switch (MI.getOpcode()) {
2166 case TargetOpcode::IMPLICIT_DEF:
2167 case TargetOpcode::KILL:
2168 case TargetOpcode::LIFETIME_START:
2169 case TargetOpcode::LIFETIME_END:
2170 return outliner::InstrType::Invisible;
2171 default:
2172 break;
2173 }
2174
2175 // Is this a terminator for a basic block?
2176 if (MI.isTerminator()) {
2177 // If this is a branch to another block, we can't outline it.
2178 if (!MI.getParent()->succ_empty())
2179 return outliner::InstrType::Illegal;
2180
2181 // Don't outline if the branch is not unconditional.
2182 if (isPredicated(MI))
2183 return outliner::InstrType::Illegal;
2184 }
2185
2186 // Make sure none of the operands of this instruction do anything that
2187 // might break if they're moved outside their current function.
2188 // This includes MachineBasicBlock references, BlockAddressses,
2189 // Constant pool indices and jump table indices.
2190 //
2191 // A quick note on MO_TargetIndex:
2192 // This doesn't seem to be used in any of the architectures that the
2193 // MachineOutliner supports, but it was still filtered out in all of them.
2194 // There was one exception (RISC-V), but MO_TargetIndex also isn't used there.
2195 // As such, this check is removed both here and in the target-specific
2196 // implementations. Instead, we assert to make sure this doesn't
2197 // catch anyone off-guard somewhere down the line.
2198 for (const MachineOperand &MOP : MI.operands()) {
2199 // If you hit this assertion, please remove it and adjust
2200 // `getOutliningTypeImpl` for your target appropriately if necessary.
2201 // Adding the assertion back to other supported architectures
2202 // would be nice too :)
2203 assert(!MOP.isTargetIndex() && "This isn't used quite yet!");
2204
2205 // CFI instructions should already have been filtered out at this point.
2206 assert(!MOP.isCFIIndex() && "CFI instructions handled elsewhere!");
2207
2208 // PrologEpilogInserter should've already run at this point.
2209 assert(!MOP.isFI() && "FrameIndex instructions should be gone by now!");
2210
2211 if (MOP.isMBB() || MOP.isBlockAddress() || MOP.isCPI() || MOP.isJTI())
2212 return outliner::InstrType::Illegal;
2213 }
2214
2215 // If we don't know, delegate to the target-specific hook.
2216 return getOutliningTypeImpl(MMI, MIT, Flags);
2217}
2218
2219bool TargetInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
2220 unsigned &Flags) const {
2221 // Some instrumentations create special TargetOpcode at the start which
2222 // expands to special code sequences which must be present.
2223 auto First = MBB.getFirstNonDebugInstr();
2224 if (First == MBB.end())
2225 return true;
2226
2227 if (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
2228 First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)
2229 return false;
2230
2231 // Some instrumentations create special pseudo-instructions at or just before
2232 // the end that must be present.
2233 auto Last = MBB.getLastNonDebugInstr();
2234 if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET ||
2235 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2236 return false;
2237
2238 if (Last != First && Last->isReturn()) {
2239 Last = prev_nodbg(It: Last, Begin: First, /*SkipPseudoOp=*/false);
2240 if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT ||
2241 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2242 return false;
2243 }
2244 return true;
2245}
2246
2247bool TargetInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
2248 return MI->isCall() || MI->hasUnmodeledSideEffects() ||
2249 (MI->hasOrderedMemoryRef() && !MI->isDereferenceableInvariantLoad());
2250}
2251