1//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetInstrInfo.h"
14#include "llvm/ADT/SmallSet.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/Dwarf.h"
17#include "llvm/CodeGen/MachineCombinerPattern.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineMemOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/MachineScheduler.h"
23#include "llvm/CodeGen/MachineTraceMetrics.h"
24#include "llvm/CodeGen/PseudoSourceValue.h"
25#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
26#include "llvm/CodeGen/StackMaps.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/CodeGen/TargetLowering.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/CodeGen/TargetSchedule.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/MC/MCAsmInfo.h"
34#include "llvm/MC/MCInstrItineraries.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Target/TargetMachine.h"
39
40using namespace llvm;
41
42static cl::opt<bool> DisableHazardRecognizer(
43 "disable-sched-hazard", cl::Hidden, cl::init(Val: false),
44 cl::desc("Disable hazard detection during preRA scheduling"));
45
46static cl::opt<bool> EnableAccReassociation(
47 "acc-reassoc", cl::Hidden, cl::init(Val: true),
48 cl::desc("Enable reassociation of accumulation chains"));
49
50static cl::opt<unsigned int>
51 MinAccumulatorDepth("acc-min-depth", cl::Hidden, cl::init(Val: 8),
52 cl::desc("Minimum length of accumulator chains "
53 "required for the optimization to kick in"));
54
55static cl::opt<unsigned int> MaxAccumulatorWidth(
56 "acc-max-width", cl::Hidden, cl::init(Val: 3),
57 cl::desc("Maximum number of branches in the accumulator tree"));
58
59TargetInstrInfo::~TargetInstrInfo() = default;
60
61const TargetRegisterClass*
62TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
63 const TargetRegisterInfo *TRI,
64 const MachineFunction &MF) const {
65 if (OpNum >= MCID.getNumOperands())
66 return nullptr;
67
68 short RegClass = MCID.operands()[OpNum].RegClass;
69 if (MCID.operands()[OpNum].isLookupPtrRegClass())
70 return TRI->getPointerRegClass(MF, Kind: RegClass);
71
72 // Instructions like INSERT_SUBREG do not have fixed register classes.
73 if (RegClass < 0)
74 return nullptr;
75
76 // Otherwise just look it up normally.
77 return TRI->getRegClass(i: RegClass);
78}
79
80/// insertNoop - Insert a noop into the instruction stream at the specified
81/// point.
82void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB,
83 MachineBasicBlock::iterator MI) const {
84 llvm_unreachable("Target didn't implement insertNoop!");
85}
86
87/// insertNoops - Insert noops into the instruction stream at the specified
88/// point.
89void TargetInstrInfo::insertNoops(MachineBasicBlock &MBB,
90 MachineBasicBlock::iterator MI,
91 unsigned Quantity) const {
92 for (unsigned i = 0; i < Quantity; ++i)
93 insertNoop(MBB, MI);
94}
95
96static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
97 return strncmp(s1: Str, s2: MAI.getCommentString().data(),
98 n: MAI.getCommentString().size()) == 0;
99}
100
101/// Measure the specified inline asm to determine an approximation of its
102/// length.
103/// Comments (which run till the next SeparatorString or newline) do not
104/// count as an instruction.
105/// Any other non-whitespace text is considered an instruction, with
106/// multiple instructions separated by SeparatorString or newlines.
107/// Variable-length instructions are not handled here; this function
108/// may be overloaded in the target code to do that.
109/// We implement a special case of the .space directive which takes only a
110/// single integer argument in base 10 that is the size in bytes. This is a
111/// restricted form of the GAS directive in that we only interpret
112/// simple--i.e. not a logical or arithmetic expression--size values without
113/// the optional fill value. This is primarily used for creating arbitrary
114/// sized inline asm blocks for testing purposes.
115unsigned TargetInstrInfo::getInlineAsmLength(
116 const char *Str,
117 const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const {
118 // Count the number of instructions in the asm.
119 bool AtInsnStart = true;
120 unsigned Length = 0;
121 const unsigned MaxInstLength = MAI.getMaxInstLength(STI);
122 for (; *Str; ++Str) {
123 if (*Str == '\n' || strncmp(s1: Str, s2: MAI.getSeparatorString(),
124 n: strlen(s: MAI.getSeparatorString())) == 0) {
125 AtInsnStart = true;
126 } else if (isAsmComment(Str, MAI)) {
127 // Stop counting as an instruction after a comment until the next
128 // separator.
129 AtInsnStart = false;
130 }
131
132 if (AtInsnStart && !isSpace(C: static_cast<unsigned char>(*Str))) {
133 unsigned AddLength = MaxInstLength;
134 if (strncmp(s1: Str, s2: ".space", n: 6) == 0) {
135 char *EStr;
136 int SpaceSize;
137 SpaceSize = strtol(nptr: Str + 6, endptr: &EStr, base: 10);
138 SpaceSize = SpaceSize < 0 ? 0 : SpaceSize;
139 while (*EStr != '\n' && isSpace(C: static_cast<unsigned char>(*EStr)))
140 ++EStr;
141 if (*EStr == '\0' || *EStr == '\n' ||
142 isAsmComment(Str: EStr, MAI)) // Successfully parsed .space argument
143 AddLength = SpaceSize;
144 }
145 Length += AddLength;
146 AtInsnStart = false;
147 }
148 }
149
150 return Length;
151}
152
153/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
154/// after it, replacing it with an unconditional branch to NewDest.
155void
156TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
157 MachineBasicBlock *NewDest) const {
158 MachineBasicBlock *MBB = Tail->getParent();
159
160 // Remove all the old successors of MBB from the CFG.
161 while (!MBB->succ_empty())
162 MBB->removeSuccessor(I: MBB->succ_begin());
163
164 // Save off the debug loc before erasing the instruction.
165 DebugLoc DL = Tail->getDebugLoc();
166
167 // Update call info and remove all the dead instructions
168 // from the end of MBB.
169 while (Tail != MBB->end()) {
170 auto MI = Tail++;
171 if (MI->shouldUpdateAdditionalCallInfo())
172 MBB->getParent()->eraseAdditionalCallInfo(MI: &*MI);
173 MBB->erase(I: MI);
174 }
175
176 // If MBB isn't immediately before MBB, insert a branch to it.
177 if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
178 insertBranch(MBB&: *MBB, TBB: NewDest, FBB: nullptr, Cond: SmallVector<MachineOperand, 0>(), DL);
179 MBB->addSuccessor(Succ: NewDest);
180}
181
182MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI,
183 bool NewMI, unsigned Idx1,
184 unsigned Idx2) const {
185 const MCInstrDesc &MCID = MI.getDesc();
186 bool HasDef = MCID.getNumDefs();
187 if (HasDef && !MI.getOperand(i: 0).isReg())
188 // No idea how to commute this instruction. Target should implement its own.
189 return nullptr;
190
191 unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
192 unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
193 assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
194 CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
195 "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
196 assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
197 "This only knows how to commute register operands so far");
198
199 Register Reg0 = HasDef ? MI.getOperand(i: 0).getReg() : Register();
200 Register Reg1 = MI.getOperand(i: Idx1).getReg();
201 Register Reg2 = MI.getOperand(i: Idx2).getReg();
202 unsigned SubReg0 = HasDef ? MI.getOperand(i: 0).getSubReg() : 0;
203 unsigned SubReg1 = MI.getOperand(i: Idx1).getSubReg();
204 unsigned SubReg2 = MI.getOperand(i: Idx2).getSubReg();
205 bool Reg1IsKill = MI.getOperand(i: Idx1).isKill();
206 bool Reg2IsKill = MI.getOperand(i: Idx2).isKill();
207 bool Reg1IsUndef = MI.getOperand(i: Idx1).isUndef();
208 bool Reg2IsUndef = MI.getOperand(i: Idx2).isUndef();
209 bool Reg1IsInternal = MI.getOperand(i: Idx1).isInternalRead();
210 bool Reg2IsInternal = MI.getOperand(i: Idx2).isInternalRead();
211 // Avoid calling isRenamable for virtual registers since we assert that
212 // renamable property is only queried/set for physical registers.
213 bool Reg1IsRenamable =
214 Reg1.isPhysical() ? MI.getOperand(i: Idx1).isRenamable() : false;
215 bool Reg2IsRenamable =
216 Reg2.isPhysical() ? MI.getOperand(i: Idx2).isRenamable() : false;
217 // If destination is tied to either of the commuted source register, then
218 // it must be updated.
219 if (HasDef && Reg0 == Reg1 &&
220 MI.getDesc().getOperandConstraint(OpNum: Idx1, Constraint: MCOI::TIED_TO) == 0) {
221 Reg2IsKill = false;
222 Reg0 = Reg2;
223 SubReg0 = SubReg2;
224 } else if (HasDef && Reg0 == Reg2 &&
225 MI.getDesc().getOperandConstraint(OpNum: Idx2, Constraint: MCOI::TIED_TO) == 0) {
226 Reg1IsKill = false;
227 Reg0 = Reg1;
228 SubReg0 = SubReg1;
229 }
230
231 MachineInstr *CommutedMI = nullptr;
232 if (NewMI) {
233 // Create a new instruction.
234 MachineFunction &MF = *MI.getMF();
235 CommutedMI = MF.CloneMachineInstr(Orig: &MI);
236 } else {
237 CommutedMI = &MI;
238 }
239
240 if (HasDef) {
241 CommutedMI->getOperand(i: 0).setReg(Reg0);
242 CommutedMI->getOperand(i: 0).setSubReg(SubReg0);
243 }
244 CommutedMI->getOperand(i: Idx2).setReg(Reg1);
245 CommutedMI->getOperand(i: Idx1).setReg(Reg2);
246 CommutedMI->getOperand(i: Idx2).setSubReg(SubReg1);
247 CommutedMI->getOperand(i: Idx1).setSubReg(SubReg2);
248 CommutedMI->getOperand(i: Idx2).setIsKill(Reg1IsKill);
249 CommutedMI->getOperand(i: Idx1).setIsKill(Reg2IsKill);
250 CommutedMI->getOperand(i: Idx2).setIsUndef(Reg1IsUndef);
251 CommutedMI->getOperand(i: Idx1).setIsUndef(Reg2IsUndef);
252 CommutedMI->getOperand(i: Idx2).setIsInternalRead(Reg1IsInternal);
253 CommutedMI->getOperand(i: Idx1).setIsInternalRead(Reg2IsInternal);
254 // Avoid calling setIsRenamable for virtual registers since we assert that
255 // renamable property is only queried/set for physical registers.
256 if (Reg1.isPhysical())
257 CommutedMI->getOperand(i: Idx2).setIsRenamable(Reg1IsRenamable);
258 if (Reg2.isPhysical())
259 CommutedMI->getOperand(i: Idx1).setIsRenamable(Reg2IsRenamable);
260 return CommutedMI;
261}
262
263MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI,
264 unsigned OpIdx1,
265 unsigned OpIdx2) const {
266 // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
267 // any commutable operand, which is done in findCommutedOpIndices() method
268 // called below.
269 if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) &&
270 !findCommutedOpIndices(MI, SrcOpIdx1&: OpIdx1, SrcOpIdx2&: OpIdx2)) {
271 assert(MI.isCommutable() &&
272 "Precondition violation: MI must be commutable.");
273 return nullptr;
274 }
275 return commuteInstructionImpl(MI, NewMI, Idx1: OpIdx1, Idx2: OpIdx2);
276}
277
278bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
279 unsigned &ResultIdx2,
280 unsigned CommutableOpIdx1,
281 unsigned CommutableOpIdx2) {
282 if (ResultIdx1 == CommuteAnyOperandIndex &&
283 ResultIdx2 == CommuteAnyOperandIndex) {
284 ResultIdx1 = CommutableOpIdx1;
285 ResultIdx2 = CommutableOpIdx2;
286 } else if (ResultIdx1 == CommuteAnyOperandIndex) {
287 if (ResultIdx2 == CommutableOpIdx1)
288 ResultIdx1 = CommutableOpIdx2;
289 else if (ResultIdx2 == CommutableOpIdx2)
290 ResultIdx1 = CommutableOpIdx1;
291 else
292 return false;
293 } else if (ResultIdx2 == CommuteAnyOperandIndex) {
294 if (ResultIdx1 == CommutableOpIdx1)
295 ResultIdx2 = CommutableOpIdx2;
296 else if (ResultIdx1 == CommutableOpIdx2)
297 ResultIdx2 = CommutableOpIdx1;
298 else
299 return false;
300 } else
301 // Check that the result operand indices match the given commutable
302 // operand indices.
303 return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) ||
304 (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1);
305
306 return true;
307}
308
309bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
310 unsigned &SrcOpIdx1,
311 unsigned &SrcOpIdx2) const {
312 assert(!MI.isBundle() &&
313 "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
314
315 const MCInstrDesc &MCID = MI.getDesc();
316 if (!MCID.isCommutable())
317 return false;
318
319 // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
320 // is not true, then the target must implement this.
321 unsigned CommutableOpIdx1 = MCID.getNumDefs();
322 unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
323 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2,
324 CommutableOpIdx1, CommutableOpIdx2))
325 return false;
326
327 if (!MI.getOperand(i: SrcOpIdx1).isReg() || !MI.getOperand(i: SrcOpIdx2).isReg())
328 // No idea.
329 return false;
330 return true;
331}
332
333bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
334 if (!MI.isTerminator()) return false;
335
336 // Conditional branch is a special case.
337 if (MI.isBranch() && !MI.isBarrier())
338 return true;
339 if (!MI.isPredicable())
340 return true;
341 return !isPredicated(MI);
342}
343
344bool TargetInstrInfo::PredicateInstruction(
345 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
346 bool MadeChange = false;
347
348 assert(!MI.isBundle() &&
349 "TargetInstrInfo::PredicateInstruction() can't handle bundles");
350
351 const MCInstrDesc &MCID = MI.getDesc();
352 if (!MI.isPredicable())
353 return false;
354
355 for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
356 if (MCID.operands()[i].isPredicate()) {
357 MachineOperand &MO = MI.getOperand(i);
358 if (MO.isReg()) {
359 MO.setReg(Pred[j].getReg());
360 MadeChange = true;
361 } else if (MO.isImm()) {
362 MO.setImm(Pred[j].getImm());
363 MadeChange = true;
364 } else if (MO.isMBB()) {
365 MO.setMBB(Pred[j].getMBB());
366 MadeChange = true;
367 }
368 ++j;
369 }
370 }
371 return MadeChange;
372}
373
374bool TargetInstrInfo::hasLoadFromStackSlot(
375 const MachineInstr &MI,
376 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
377 size_t StartSize = Accesses.size();
378 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
379 oe = MI.memoperands_end();
380 o != oe; ++o) {
381 if ((*o)->isLoad() &&
382 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
383 Accesses.push_back(Elt: *o);
384 }
385 return Accesses.size() != StartSize;
386}
387
388bool TargetInstrInfo::hasStoreToStackSlot(
389 const MachineInstr &MI,
390 SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
391 size_t StartSize = Accesses.size();
392 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
393 oe = MI.memoperands_end();
394 o != oe; ++o) {
395 if ((*o)->isStore() &&
396 isa_and_nonnull<FixedStackPseudoSourceValue>(Val: (*o)->getPseudoValue()))
397 Accesses.push_back(Elt: *o);
398 }
399 return Accesses.size() != StartSize;
400}
401
402bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
403 unsigned SubIdx, unsigned &Size,
404 unsigned &Offset,
405 const MachineFunction &MF) const {
406 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
407 if (!SubIdx) {
408 Size = TRI->getSpillSize(RC: *RC);
409 Offset = 0;
410 return true;
411 }
412 unsigned BitSize = TRI->getSubRegIdxSize(Idx: SubIdx);
413 // Convert bit size to byte size.
414 if (BitSize % 8)
415 return false;
416
417 int BitOffset = TRI->getSubRegIdxOffset(Idx: SubIdx);
418 if (BitOffset < 0 || BitOffset % 8)
419 return false;
420
421 Size = BitSize / 8;
422 Offset = (unsigned)BitOffset / 8;
423
424 assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
425
426 if (!MF.getDataLayout().isLittleEndian()) {
427 Offset = TRI->getSpillSize(RC: *RC) - (Offset + Size);
428 }
429 return true;
430}
431
432void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
433 MachineBasicBlock::iterator I,
434 Register DestReg, unsigned SubIdx,
435 const MachineInstr &Orig,
436 const TargetRegisterInfo &TRI) const {
437 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
438 MI->substituteRegister(FromReg: MI->getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
439 MBB.insert(I, MI);
440}
441
442bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0,
443 const MachineInstr &MI1,
444 const MachineRegisterInfo *MRI) const {
445 return MI0.isIdenticalTo(Other: MI1, Check: MachineInstr::IgnoreVRegDefs);
446}
447
448MachineInstr &
449TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
450 MachineBasicBlock::iterator InsertBefore,
451 const MachineInstr &Orig) const {
452 MachineFunction &MF = *MBB.getParent();
453 // CFI instructions are marked as non-duplicable, because Darwin compact
454 // unwind info emission can't handle multiple prologue setups.
455 assert((!Orig.isNotDuplicable() ||
456 (!MF.getTarget().getTargetTriple().isOSDarwin() &&
457 Orig.isCFIInstruction())) &&
458 "Instruction cannot be duplicated");
459
460 return MF.cloneMachineInstrBundle(MBB, InsertBefore, Orig);
461}
462
463// If the COPY instruction in MI can be folded to a stack operation, return
464// the register class to use.
465static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI,
466 const TargetInstrInfo &TII,
467 unsigned FoldIdx) {
468 assert(TII.isCopyInstr(MI) && "MI must be a COPY instruction");
469 if (MI.getNumOperands() != 2)
470 return nullptr;
471 assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
472
473 const MachineOperand &FoldOp = MI.getOperand(i: FoldIdx);
474 const MachineOperand &LiveOp = MI.getOperand(i: 1 - FoldIdx);
475
476 if (FoldOp.getSubReg() || LiveOp.getSubReg())
477 return nullptr;
478
479 Register FoldReg = FoldOp.getReg();
480 Register LiveReg = LiveOp.getReg();
481
482 assert(FoldReg.isVirtual() && "Cannot fold physregs");
483
484 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
485 const TargetRegisterClass *RC = MRI.getRegClass(Reg: FoldReg);
486
487 if (LiveOp.getReg().isPhysical())
488 return RC->contains(Reg: LiveOp.getReg()) ? RC : nullptr;
489
490 if (RC->hasSubClassEq(RC: MRI.getRegClass(Reg: LiveReg)))
491 return RC;
492
493 // FIXME: Allow folding when register classes are memory compatible.
494 return nullptr;
495}
496
497MCInst TargetInstrInfo::getNop() const { llvm_unreachable("Not implemented"); }
498
499/// Try to remove the load by folding it to a register
500/// operand at the use. We fold the load instructions if load defines a virtual
501/// register, the virtual register is used once in the same BB, and the
502/// instructions in-between do not load or store, and have no side effects.
503MachineInstr *TargetInstrInfo::optimizeLoadInstr(MachineInstr &MI,
504 const MachineRegisterInfo *MRI,
505 Register &FoldAsLoadDefReg,
506 MachineInstr *&DefMI) const {
507 // Check whether we can move DefMI here.
508 DefMI = MRI->getVRegDef(Reg: FoldAsLoadDefReg);
509 assert(DefMI);
510 bool SawStore = false;
511 if (!DefMI->isSafeToMove(SawStore))
512 return nullptr;
513
514 // Collect information about virtual register operands of MI.
515 SmallVector<unsigned, 1> SrcOperandIds;
516 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
517 MachineOperand &MO = MI.getOperand(i);
518 if (!MO.isReg())
519 continue;
520 Register Reg = MO.getReg();
521 if (Reg != FoldAsLoadDefReg)
522 continue;
523 // Do not fold if we have a subreg use or a def.
524 if (MO.getSubReg() || MO.isDef())
525 return nullptr;
526 SrcOperandIds.push_back(Elt: i);
527 }
528 if (SrcOperandIds.empty())
529 return nullptr;
530
531 // Check whether we can fold the def into SrcOperandId.
532 if (MachineInstr *FoldMI = foldMemoryOperand(MI, Ops: SrcOperandIds, LoadMI&: *DefMI)) {
533 FoldAsLoadDefReg = 0;
534 return FoldMI;
535 }
536
537 return nullptr;
538}
539
540std::pair<unsigned, unsigned>
541TargetInstrInfo::getPatchpointUnfoldableRange(const MachineInstr &MI) const {
542 switch (MI.getOpcode()) {
543 case TargetOpcode::STACKMAP:
544 // StackMapLiveValues are foldable
545 return std::make_pair(x: 0, y: StackMapOpers(&MI).getVarIdx());
546 case TargetOpcode::PATCHPOINT:
547 // For PatchPoint, the call args are not foldable (even if reported in the
548 // stackmap e.g. via anyregcc).
549 return std::make_pair(x: 0, y: PatchPointOpers(&MI).getVarIdx());
550 case TargetOpcode::STATEPOINT:
551 // For statepoints, fold deopt and gc arguments, but not call arguments.
552 return std::make_pair(x: MI.getNumDefs(), y: StatepointOpers(&MI).getVarIdx());
553 default:
554 llvm_unreachable("unexpected stackmap opcode");
555 }
556}
557
558static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
559 ArrayRef<unsigned> Ops, int FrameIndex,
560 const TargetInstrInfo &TII) {
561 unsigned StartIdx = 0;
562 unsigned NumDefs = 0;
563 // getPatchpointUnfoldableRange throws guarantee if MI is not a patchpoint.
564 std::tie(args&: NumDefs, args&: StartIdx) = TII.getPatchpointUnfoldableRange(MI);
565
566 unsigned DefToFoldIdx = MI.getNumOperands();
567
568 // Return false if any operands requested for folding are not foldable (not
569 // part of the stackmap's live values).
570 for (unsigned Op : Ops) {
571 if (Op < NumDefs) {
572 assert(DefToFoldIdx == MI.getNumOperands() && "Folding multiple defs");
573 DefToFoldIdx = Op;
574 } else if (Op < StartIdx) {
575 return nullptr;
576 }
577 if (MI.getOperand(i: Op).isTied())
578 return nullptr;
579 }
580
581 MachineInstr *NewMI =
582 MF.CreateMachineInstr(MCID: TII.get(Opcode: MI.getOpcode()), DL: MI.getDebugLoc(), NoImplicit: true);
583 MachineInstrBuilder MIB(MF, NewMI);
584
585 // No need to fold return, the meta data, and function arguments
586 for (unsigned i = 0; i < StartIdx; ++i)
587 if (i != DefToFoldIdx)
588 MIB.add(MO: MI.getOperand(i));
589
590 for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) {
591 MachineOperand &MO = MI.getOperand(i);
592 unsigned TiedTo = e;
593 (void)MI.isRegTiedToDefOperand(UseOpIdx: i, DefOpIdx: &TiedTo);
594
595 if (is_contained(Range&: Ops, Element: i)) {
596 assert(TiedTo == e && "Cannot fold tied operands");
597 unsigned SpillSize;
598 unsigned SpillOffset;
599 // Compute the spill slot size and offset.
600 const TargetRegisterClass *RC =
601 MF.getRegInfo().getRegClass(Reg: MO.getReg());
602 bool Valid =
603 TII.getStackSlotRange(RC, SubIdx: MO.getSubReg(), Size&: SpillSize, Offset&: SpillOffset, MF);
604 if (!Valid)
605 report_fatal_error(reason: "cannot spill patchpoint subregister operand");
606 MIB.addImm(Val: StackMaps::IndirectMemRefOp);
607 MIB.addImm(Val: SpillSize);
608 MIB.addFrameIndex(Idx: FrameIndex);
609 MIB.addImm(Val: SpillOffset);
610 } else {
611 MIB.add(MO);
612 if (TiedTo < e) {
613 assert(TiedTo < NumDefs && "Bad tied operand");
614 if (TiedTo > DefToFoldIdx)
615 --TiedTo;
616 NewMI->tieOperands(DefIdx: TiedTo, UseIdx: NewMI->getNumOperands() - 1);
617 }
618 }
619 }
620 return NewMI;
621}
622
623static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI,
624 const TargetInstrInfo &TII) {
625 // If the machine operand is tied, untie it first.
626 if (MI->getOperand(i: OpNo).isTied()) {
627 unsigned TiedTo = MI->findTiedOperandIdx(OpIdx: OpNo);
628 MI->untieRegOperand(OpIdx: OpNo);
629 // Intentional recursion!
630 foldInlineAsmMemOperand(MI, OpNo: TiedTo, FI, TII);
631 }
632
633 SmallVector<MachineOperand, 5> NewOps;
634 TII.getFrameIndexOperands(Ops&: NewOps, FI);
635 assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands");
636 MI->removeOperand(OpNo);
637 MI->insert(InsertBefore: MI->operands_begin() + OpNo, Ops: NewOps);
638
639 // Change the previous operand to a MemKind InlineAsm::Flag. The second param
640 // is the per-target number of operands that represent the memory operand
641 // excluding this one (MD). This includes MO.
642 InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size());
643 F.setMemConstraint(InlineAsm::ConstraintCode::m);
644 MachineOperand &MD = MI->getOperand(i: OpNo - 1);
645 MD.setImm(F);
646}
647
648// Returns nullptr if not possible to fold.
649static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
650 ArrayRef<unsigned> Ops, int FI,
651 const TargetInstrInfo &TII) {
652 assert(MI.isInlineAsm() && "wrong opcode");
653 if (Ops.size() > 1)
654 return nullptr;
655 unsigned Op = Ops[0];
656 assert(Op && "should never be first operand");
657 assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands");
658
659 if (!MI.mayFoldInlineAsmRegOp(OpId: Op))
660 return nullptr;
661
662 MachineInstr &NewMI = TII.duplicate(MBB&: *MI.getParent(), InsertBefore: MI.getIterator(), Orig: MI);
663
664 foldInlineAsmMemOperand(MI: &NewMI, OpNo: Op, FI, TII);
665
666 // Update mayload/maystore metadata, and memoperands.
667 const VirtRegInfo &RI =
668 AnalyzeVirtRegInBundle(MI, Reg: MI.getOperand(i: Op).getReg());
669 MachineOperand &ExtraMO = NewMI.getOperand(i: InlineAsm::MIOp_ExtraInfo);
670 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
671 if (RI.Reads) {
672 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad);
673 Flags |= MachineMemOperand::MOLoad;
674 }
675 if (RI.Writes) {
676 ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
677 Flags |= MachineMemOperand::MOStore;
678 }
679 MachineFunction *MF = NewMI.getMF();
680 const MachineFrameInfo &MFI = MF->getFrameInfo();
681 MachineMemOperand *MMO = MF->getMachineMemOperand(
682 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: Flags, Size: MFI.getObjectSize(ObjectIdx: FI),
683 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
684 NewMI.addMemOperand(MF&: *MF, MO: MMO);
685
686 return &NewMI;
687}
688
689MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
690 ArrayRef<unsigned> Ops, int FI,
691 LiveIntervals *LIS,
692 VirtRegMap *VRM) const {
693 auto Flags = MachineMemOperand::MONone;
694 for (unsigned OpIdx : Ops)
695 Flags |= MI.getOperand(i: OpIdx).isDef() ? MachineMemOperand::MOStore
696 : MachineMemOperand::MOLoad;
697
698 MachineBasicBlock *MBB = MI.getParent();
699 assert(MBB && "foldMemoryOperand needs an inserted instruction");
700 MachineFunction &MF = *MBB->getParent();
701
702 // If we're not folding a load into a subreg, the size of the load is the
703 // size of the spill slot. But if we are, we need to figure out what the
704 // actual load size is.
705 int64_t MemSize = 0;
706 const MachineFrameInfo &MFI = MF.getFrameInfo();
707 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
708
709 if (Flags & MachineMemOperand::MOStore) {
710 MemSize = MFI.getObjectSize(ObjectIdx: FI);
711 } else {
712 for (unsigned OpIdx : Ops) {
713 int64_t OpSize = MFI.getObjectSize(ObjectIdx: FI);
714
715 if (auto SubReg = MI.getOperand(i: OpIdx).getSubReg()) {
716 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: SubReg);
717 if (SubRegSize > 0 && !(SubRegSize % 8))
718 OpSize = SubRegSize / 8;
719 }
720
721 MemSize = std::max(a: MemSize, b: OpSize);
722 }
723 }
724
725 assert(MemSize && "Did not expect a zero-sized stack slot");
726
727 MachineInstr *NewMI = nullptr;
728
729 if (MI.getOpcode() == TargetOpcode::STACKMAP ||
730 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
731 MI.getOpcode() == TargetOpcode::STATEPOINT) {
732 // Fold stackmap/patchpoint.
733 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex: FI, TII: *this);
734 if (NewMI)
735 MBB->insert(I: MI, MI: NewMI);
736 } else if (MI.isInlineAsm()) {
737 return foldInlineAsmMemOperand(MI, Ops, FI, TII: *this);
738 } else {
739 // Ask the target to do the actual folding.
740 NewMI = foldMemoryOperandImpl(MF, MI, Ops, InsertPt: MI, FrameIndex: FI, LIS, VRM);
741 }
742
743 if (NewMI) {
744 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
745 // Add a memory operand, foldMemoryOperandImpl doesn't do that.
746 assert((!(Flags & MachineMemOperand::MOStore) ||
747 NewMI->mayStore()) &&
748 "Folded a def to a non-store!");
749 assert((!(Flags & MachineMemOperand::MOLoad) ||
750 NewMI->mayLoad()) &&
751 "Folded a use to a non-load!");
752 assert(MFI.getObjectOffset(FI) != -1);
753 MachineMemOperand *MMO =
754 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF, FI),
755 F: Flags, Size: MemSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
756 NewMI->addMemOperand(MF, MO: MMO);
757
758 // The pass "x86 speculative load hardening" always attaches symbols to
759 // call instructions. We need copy it form old instruction.
760 NewMI->cloneInstrSymbols(MF, MI);
761
762 return NewMI;
763 }
764
765 // Straight COPY may fold as load/store.
766 if (!isCopyInstr(MI) || Ops.size() != 1)
767 return nullptr;
768
769 const TargetRegisterClass *RC = canFoldCopy(MI, TII: *this, FoldIdx: Ops[0]);
770 if (!RC)
771 return nullptr;
772
773 const MachineOperand &MO = MI.getOperand(i: 1 - Ops[0]);
774 MachineBasicBlock::iterator Pos = MI;
775
776 if (Flags == MachineMemOperand::MOStore)
777 storeRegToStackSlot(MBB&: *MBB, MI: Pos, SrcReg: MO.getReg(), isKill: MO.isKill(), FrameIndex: FI, RC, TRI,
778 VReg: Register());
779 else
780 loadRegFromStackSlot(MBB&: *MBB, MI: Pos, DestReg: MO.getReg(), FrameIndex: FI, RC, TRI, VReg: Register());
781 return &*--Pos;
782}
783
784MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
785 ArrayRef<unsigned> Ops,
786 MachineInstr &LoadMI,
787 LiveIntervals *LIS) const {
788 assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
789#ifndef NDEBUG
790 for (unsigned OpIdx : Ops)
791 assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
792#endif
793
794 MachineBasicBlock &MBB = *MI.getParent();
795 MachineFunction &MF = *MBB.getParent();
796
797 // Ask the target to do the actual folding.
798 MachineInstr *NewMI = nullptr;
799 int FrameIndex = 0;
800
801 if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
802 MI.getOpcode() == TargetOpcode::PATCHPOINT ||
803 MI.getOpcode() == TargetOpcode::STATEPOINT) &&
804 isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
805 // Fold stackmap/patchpoint.
806 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, TII: *this);
807 if (NewMI)
808 NewMI = &*MBB.insert(I: MI, MI: NewMI);
809 } else if (MI.isInlineAsm() && isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
810 return foldInlineAsmMemOperand(MI, Ops, FI: FrameIndex, TII: *this);
811 } else {
812 // Ask the target to do the actual folding.
813 NewMI = foldMemoryOperandImpl(MF, MI, Ops, InsertPt: MI, LoadMI, LIS);
814 }
815
816 if (!NewMI)
817 return nullptr;
818
819 // Copy the memoperands from the load to the folded instruction.
820 if (MI.memoperands_empty()) {
821 NewMI->setMemRefs(MF, MemRefs: LoadMI.memoperands());
822 } else {
823 // Handle the rare case of folding multiple loads.
824 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
825 for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(),
826 E = LoadMI.memoperands_end();
827 I != E; ++I) {
828 NewMI->addMemOperand(MF, MO: *I);
829 }
830 }
831 return NewMI;
832}
833
834/// transferImplicitOperands - MI is a pseudo-instruction, and the lowered
835/// replacement instructions immediately precede it. Copy any implicit
836/// operands from MI to the replacement instruction.
837static void transferImplicitOperands(MachineInstr *MI,
838 const TargetRegisterInfo *TRI) {
839 MachineBasicBlock::iterator CopyMI = MI;
840 --CopyMI;
841
842 Register DstReg = MI->getOperand(i: 0).getReg();
843 for (const MachineOperand &MO : MI->implicit_operands()) {
844 CopyMI->addOperand(Op: MO);
845
846 // Be conservative about preserving kills when subregister defs are
847 // involved. If there was implicit kill of a super-register overlapping the
848 // copy result, we would kill the subregisters previous copies defined.
849
850 if (MO.isKill() && TRI->regsOverlap(RegA: DstReg, RegB: MO.getReg()))
851 CopyMI->getOperand(i: CopyMI->getNumOperands() - 1).setIsKill(false);
852 }
853}
854
855void TargetInstrInfo::lowerCopy(MachineInstr *MI,
856 const TargetRegisterInfo *TRI) const {
857 if (MI->allDefsAreDead()) {
858 MI->setDesc(get(Opcode: TargetOpcode::KILL));
859 return;
860 }
861
862 MachineOperand &DstMO = MI->getOperand(i: 0);
863 MachineOperand &SrcMO = MI->getOperand(i: 1);
864
865 bool IdentityCopy = (SrcMO.getReg() == DstMO.getReg());
866 if (IdentityCopy || SrcMO.isUndef()) {
867 // No need to insert an identity copy instruction, but replace with a KILL
868 // if liveness is changed.
869 if (SrcMO.isUndef() || MI->getNumOperands() > 2) {
870 // We must make sure the super-register gets killed. Replace the
871 // instruction with KILL.
872 MI->setDesc(get(Opcode: TargetOpcode::KILL));
873 return;
874 }
875 // Vanilla identity copy.
876 MI->eraseFromParent();
877 return;
878 }
879
880 copyPhysReg(MBB&: *MI->getParent(), MI, DL: MI->getDebugLoc(), DestReg: DstMO.getReg(),
881 SrcReg: SrcMO.getReg(), KillSrc: SrcMO.isKill(),
882 RenamableDest: DstMO.getReg().isPhysical() ? DstMO.isRenamable() : false,
883 RenamableSrc: SrcMO.getReg().isPhysical() ? SrcMO.isRenamable() : false);
884
885 if (MI->getNumOperands() > 2)
886 transferImplicitOperands(MI, TRI);
887 MI->eraseFromParent();
888}
889
890bool TargetInstrInfo::hasReassociableOperands(
891 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
892 const MachineOperand &Op1 = Inst.getOperand(i: 1);
893 const MachineOperand &Op2 = Inst.getOperand(i: 2);
894 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
895
896 // We need virtual register definitions for the operands that we will
897 // reassociate.
898 MachineInstr *MI1 = nullptr;
899 MachineInstr *MI2 = nullptr;
900 if (Op1.isReg() && Op1.getReg().isVirtual())
901 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
902 if (Op2.isReg() && Op2.getReg().isVirtual())
903 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
904
905 // And at least one operand must be defined in MBB.
906 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
907}
908
909bool TargetInstrInfo::areOpcodesEqualOrInverse(unsigned Opcode1,
910 unsigned Opcode2) const {
911 return Opcode1 == Opcode2 || getInverseOpcode(Opcode: Opcode1) == Opcode2;
912}
913
914bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
915 bool &Commuted) const {
916 const MachineBasicBlock *MBB = Inst.getParent();
917 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
918 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 1).getReg());
919 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
920 unsigned Opcode = Inst.getOpcode();
921
922 // If only one operand has the same or inverse opcode and it's the second
923 // source operand, the operands must be commuted.
924 Commuted = !areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
925 areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI2->getOpcode());
926 if (Commuted)
927 std::swap(a&: MI1, b&: MI2);
928
929 // 1. The previous instruction must be the same type as Inst.
930 // 2. The previous instruction must also be associative/commutative or be the
931 // inverse of such an operation (this can be different even for
932 // instructions with the same opcode if traits like fast-math-flags are
933 // included).
934 // 3. The previous instruction must have virtual register definitions for its
935 // operands in the same basic block as Inst.
936 // 4. The previous instruction's result must only be used by Inst.
937 return areOpcodesEqualOrInverse(Opcode1: Opcode, Opcode2: MI1->getOpcode()) &&
938 (isAssociativeAndCommutative(Inst: *MI1) ||
939 isAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
940 hasReassociableOperands(Inst: *MI1, MBB) &&
941 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
942}
943
944// 1. The operation must be associative and commutative or be the inverse of
945// such an operation.
946// 2. The instruction must have virtual register definitions for its
947// operands in the same basic block.
948// 3. The instruction must have a reassociable sibling.
949bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
950 bool &Commuted) const {
951 return (isAssociativeAndCommutative(Inst) ||
952 isAssociativeAndCommutative(Inst, /* Invert */ true)) &&
953 hasReassociableOperands(Inst, MBB: Inst.getParent()) &&
954 hasReassociableSibling(Inst, Commuted);
955}
956
957// Utility routine that checks if \param MO is defined by an
958// \param CombineOpc instruction in the basic block \param MBB.
959// If \param CombineOpc is not provided, the OpCode check will
960// be skipped.
961static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
962 unsigned CombineOpc = 0) {
963 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
964 MachineInstr *MI = nullptr;
965
966 if (MO.isReg() && MO.getReg().isVirtual())
967 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
968 // And it needs to be in the trace (otherwise, it won't have a depth).
969 if (!MI || MI->getParent() != &MBB ||
970 ((unsigned)MI->getOpcode() != CombineOpc && CombineOpc != 0))
971 return false;
972 // Must only used by the user we combine with.
973 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
974 return false;
975
976 return true;
977}
978
979// A chain of accumulation instructions will be selected IFF:
980// 1. All the accumulation instructions in the chain have the same opcode,
981// besides the first that has a slightly different opcode because it does
982// not accumulate into a register.
983// 2. All the instructions in the chain are combinable (have a single use
984// which itself is part of the chain).
985// 3. Meets the required minimum length.
986void TargetInstrInfo::getAccumulatorChain(
987 MachineInstr *CurrentInstr, SmallVectorImpl<Register> &Chain) const {
988 // Walk up the chain of accumulation instructions and collect them in the
989 // vector.
990 MachineBasicBlock &MBB = *CurrentInstr->getParent();
991 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
992 unsigned AccumulatorOpcode = CurrentInstr->getOpcode();
993 std::optional<unsigned> ChainStartOpCode =
994 getAccumulationStartOpcode(Opcode: AccumulatorOpcode);
995
996 if (!ChainStartOpCode.has_value())
997 return;
998
999 // Push the first accumulator result to the start of the chain.
1000 Chain.push_back(Elt: CurrentInstr->getOperand(i: 0).getReg());
1001
1002 // Collect the accumulator input register from all instructions in the chain.
1003 while (CurrentInstr &&
1004 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1), CombineOpc: AccumulatorOpcode)) {
1005 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1006 CurrentInstr = MRI.getUniqueVRegDef(Reg: CurrentInstr->getOperand(i: 1).getReg());
1007 }
1008
1009 // Add the instruction at the top of the chain.
1010 if (CurrentInstr->getOpcode() == AccumulatorOpcode &&
1011 canCombine(MBB, MO&: CurrentInstr->getOperand(i: 1)))
1012 Chain.push_back(Elt: CurrentInstr->getOperand(i: 1).getReg());
1013}
1014
1015/// Find chains of accumulations that can be rewritten as a tree for increased
1016/// ILP.
1017bool TargetInstrInfo::getAccumulatorReassociationPatterns(
1018 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns) const {
1019 if (!EnableAccReassociation)
1020 return false;
1021
1022 unsigned Opc = Root.getOpcode();
1023 if (!isAccumulationOpcode(Opcode: Opc))
1024 return false;
1025
1026 // Verify that this is the end of the chain.
1027 MachineBasicBlock &MBB = *Root.getParent();
1028 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1029 if (!MRI.hasOneNonDBGUser(RegNo: Root.getOperand(i: 0).getReg()))
1030 return false;
1031
1032 auto User = MRI.use_instr_begin(RegNo: Root.getOperand(i: 0).getReg());
1033 if (User->getOpcode() == Opc)
1034 return false;
1035
1036 // Walk up the use chain and collect the reduction chain.
1037 SmallVector<Register, 32> Chain;
1038 getAccumulatorChain(CurrentInstr: &Root, Chain);
1039
1040 // Reject chains which are too short to be worth modifying.
1041 if (Chain.size() < MinAccumulatorDepth)
1042 return false;
1043
1044 // Check if the MBB this instruction is a part of contains any other chains.
1045 // If so, don't apply it.
1046 SmallSet<Register, 32> ReductionChain(llvm::from_range, Chain);
1047 for (const auto &I : MBB) {
1048 if (I.getOpcode() == Opc &&
1049 !ReductionChain.contains(V: I.getOperand(i: 0).getReg()))
1050 return false;
1051 }
1052
1053 Patterns.push_back(Elt: MachineCombinerPattern::ACC_CHAIN);
1054 return true;
1055}
1056
1057// Reduce branches of the accumulator tree by adding them together.
1058void TargetInstrInfo::reduceAccumulatorTree(
1059 SmallVectorImpl<Register> &RegistersToReduce,
1060 SmallVectorImpl<MachineInstr *> &InsInstrs, MachineFunction &MF,
1061 MachineInstr &Root, MachineRegisterInfo &MRI,
1062 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
1063 Register ResultReg) const {
1064 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1065 SmallVector<Register, 8> NewRegs;
1066
1067 // Get the opcode for the reduction instruction we will need to build.
1068 // If for some reason it is not defined, early exit and don't apply this.
1069 unsigned ReduceOpCode = getReduceOpcodeForAccumulator(AccumulatorOpCode: Root.getOpcode());
1070
1071 for (unsigned int i = 1; i <= (RegistersToReduce.size() / 2); i += 2) {
1072 auto RHS = RegistersToReduce[i - 1];
1073 auto LHS = RegistersToReduce[i];
1074 Register Dest;
1075 // If we are reducing 2 registers, reuse the original result register.
1076 if (RegistersToReduce.size() == 2)
1077 Dest = ResultReg;
1078 // Otherwise, create a new virtual register to hold the partial sum.
1079 else {
1080 auto NewVR = MRI.createVirtualRegister(
1081 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()));
1082 Dest = NewVR;
1083 NewRegs.push_back(Elt: Dest);
1084 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: Dest, y: InsInstrs.size()));
1085 }
1086
1087 // Create the new reduction instruction.
1088 MachineInstrBuilder MIB =
1089 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: ReduceOpCode), DestReg: Dest)
1090 .addReg(RegNo: RHS, flags: getKillRegState(B: true))
1091 .addReg(RegNo: LHS, flags: getKillRegState(B: true));
1092 // Copy any flags needed from the original instruction.
1093 MIB->setFlags(Root.getFlags());
1094 InsInstrs.push_back(Elt: MIB);
1095 }
1096
1097 // If the number of registers to reduce is odd, add the remaining register to
1098 // the vector of registers to reduce.
1099 if (RegistersToReduce.size() % 2 != 0)
1100 NewRegs.push_back(Elt: RegistersToReduce[RegistersToReduce.size() - 1]);
1101
1102 RegistersToReduce = NewRegs;
1103}
1104
1105// The concept of the reassociation pass is that these operations can benefit
1106// from this kind of transformation:
1107//
1108// A = ? op ?
1109// B = A op X (Prev)
1110// C = B op Y (Root)
1111// -->
1112// A = ? op ?
1113// B = X op Y
1114// C = A op B
1115//
1116// breaking the dependency between A and B, allowing them to be executed in
1117// parallel (or back-to-back in a pipeline) instead of depending on each other.
1118
1119// FIXME: This has the potential to be expensive (compile time) while not
1120// improving the code at all. Some ways to limit the overhead:
1121// 1. Track successful transforms; bail out if hit rate gets too low.
1122// 2. Only enable at -O3 or some other non-default optimization level.
1123// 3. Pre-screen pattern candidates here: if an operand of the previous
1124// instruction is known to not increase the critical path, then don't match
1125// that pattern.
1126bool TargetInstrInfo::getMachineCombinerPatterns(
1127 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
1128 bool DoRegPressureReduce) const {
1129 bool Commute;
1130 if (isReassociationCandidate(Inst: Root, Commuted&: Commute)) {
1131 // We found a sequence of instructions that may be suitable for a
1132 // reassociation of operands to increase ILP. Specify each commutation
1133 // possibility for the Prev instruction in the sequence and let the
1134 // machine combiner decide if changing the operands is worthwhile.
1135 if (Commute) {
1136 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_YB);
1137 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_YB);
1138 } else {
1139 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_AX_BY);
1140 Patterns.push_back(Elt: MachineCombinerPattern::REASSOC_XA_BY);
1141 }
1142 return true;
1143 }
1144 if (getAccumulatorReassociationPatterns(Root, Patterns))
1145 return true;
1146
1147 return false;
1148}
1149
1150/// Return true when a code sequence can improve loop throughput.
1151bool TargetInstrInfo::isThroughputPattern(unsigned Pattern) const {
1152 return false;
1153}
1154
1155CombinerObjective
1156TargetInstrInfo::getCombinerObjective(unsigned Pattern) const {
1157 switch (Pattern) {
1158 case MachineCombinerPattern::ACC_CHAIN:
1159 return CombinerObjective::MustReduceDepth;
1160 default:
1161 return CombinerObjective::Default;
1162 }
1163}
1164
1165std::pair<unsigned, unsigned>
1166TargetInstrInfo::getReassociationOpcodes(unsigned Pattern,
1167 const MachineInstr &Root,
1168 const MachineInstr &Prev) const {
1169 bool AssocCommutRoot = isAssociativeAndCommutative(Inst: Root);
1170 bool AssocCommutPrev = isAssociativeAndCommutative(Inst: Prev);
1171
1172 // Early exit if both opcodes are associative and commutative. It's a trivial
1173 // reassociation when we only change operands order. In this case opcodes are
1174 // not required to have inverse versions.
1175 if (AssocCommutRoot && AssocCommutPrev) {
1176 assert(Root.getOpcode() == Prev.getOpcode() && "Expected to be equal");
1177 return std::make_pair(x: Root.getOpcode(), y: Root.getOpcode());
1178 }
1179
1180 // At least one instruction is not associative or commutative.
1181 // Since we have matched one of the reassociation patterns, we expect that the
1182 // instructions' opcodes are equal or one of them is the inversion of the
1183 // other.
1184 assert(areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()) &&
1185 "Incorrectly matched pattern");
1186 unsigned AssocCommutOpcode = Root.getOpcode();
1187 unsigned InverseOpcode = *getInverseOpcode(Opcode: Root.getOpcode());
1188 if (!AssocCommutRoot)
1189 std::swap(a&: AssocCommutOpcode, b&: InverseOpcode);
1190
1191 // The transformation rule (`+` is any associative and commutative binary
1192 // operation, `-` is the inverse):
1193 // REASSOC_AX_BY:
1194 // (A + X) + Y => A + (X + Y)
1195 // (A + X) - Y => A + (X - Y)
1196 // (A - X) + Y => A - (X - Y)
1197 // (A - X) - Y => A - (X + Y)
1198 // REASSOC_XA_BY:
1199 // (X + A) + Y => (X + Y) + A
1200 // (X + A) - Y => (X - Y) + A
1201 // (X - A) + Y => (X + Y) - A
1202 // (X - A) - Y => (X - Y) - A
1203 // REASSOC_AX_YB:
1204 // Y + (A + X) => (Y + X) + A
1205 // Y - (A + X) => (Y - X) - A
1206 // Y + (A - X) => (Y - X) + A
1207 // Y - (A - X) => (Y + X) - A
1208 // REASSOC_XA_YB:
1209 // Y + (X + A) => (Y + X) + A
1210 // Y - (X + A) => (Y - X) - A
1211 // Y + (X - A) => (Y + X) - A
1212 // Y - (X - A) => (Y - X) + A
1213 switch (Pattern) {
1214 default:
1215 llvm_unreachable("Unexpected pattern");
1216 case MachineCombinerPattern::REASSOC_AX_BY:
1217 if (!AssocCommutRoot && AssocCommutPrev)
1218 return {AssocCommutOpcode, InverseOpcode};
1219 if (AssocCommutRoot && !AssocCommutPrev)
1220 return {InverseOpcode, InverseOpcode};
1221 if (!AssocCommutRoot && !AssocCommutPrev)
1222 return {InverseOpcode, AssocCommutOpcode};
1223 break;
1224 case MachineCombinerPattern::REASSOC_XA_BY:
1225 if (!AssocCommutRoot && AssocCommutPrev)
1226 return {AssocCommutOpcode, InverseOpcode};
1227 if (AssocCommutRoot && !AssocCommutPrev)
1228 return {InverseOpcode, AssocCommutOpcode};
1229 if (!AssocCommutRoot && !AssocCommutPrev)
1230 return {InverseOpcode, InverseOpcode};
1231 break;
1232 case MachineCombinerPattern::REASSOC_AX_YB:
1233 if (!AssocCommutRoot && AssocCommutPrev)
1234 return {InverseOpcode, InverseOpcode};
1235 if (AssocCommutRoot && !AssocCommutPrev)
1236 return {AssocCommutOpcode, InverseOpcode};
1237 if (!AssocCommutRoot && !AssocCommutPrev)
1238 return {InverseOpcode, AssocCommutOpcode};
1239 break;
1240 case MachineCombinerPattern::REASSOC_XA_YB:
1241 if (!AssocCommutRoot && AssocCommutPrev)
1242 return {InverseOpcode, InverseOpcode};
1243 if (AssocCommutRoot && !AssocCommutPrev)
1244 return {InverseOpcode, AssocCommutOpcode};
1245 if (!AssocCommutRoot && !AssocCommutPrev)
1246 return {AssocCommutOpcode, InverseOpcode};
1247 break;
1248 }
1249 llvm_unreachable("Unhandled combination");
1250}
1251
1252// Return a pair of boolean flags showing if the new root and new prev operands
1253// must be swapped. See visual example of the rule in
1254// TargetInstrInfo::getReassociationOpcodes.
1255static std::pair<bool, bool> mustSwapOperands(unsigned Pattern) {
1256 switch (Pattern) {
1257 default:
1258 llvm_unreachable("Unexpected pattern");
1259 case MachineCombinerPattern::REASSOC_AX_BY:
1260 return {false, false};
1261 case MachineCombinerPattern::REASSOC_XA_BY:
1262 return {true, false};
1263 case MachineCombinerPattern::REASSOC_AX_YB:
1264 return {true, true};
1265 case MachineCombinerPattern::REASSOC_XA_YB:
1266 return {true, true};
1267 }
1268}
1269
1270void TargetInstrInfo::getReassociateOperandIndices(
1271 const MachineInstr &Root, unsigned Pattern,
1272 std::array<unsigned, 5> &OperandIndices) const {
1273 switch (Pattern) {
1274 case MachineCombinerPattern::REASSOC_AX_BY:
1275 OperandIndices = {1, 1, 1, 2, 2};
1276 break;
1277 case MachineCombinerPattern::REASSOC_AX_YB:
1278 OperandIndices = {2, 1, 2, 2, 1};
1279 break;
1280 case MachineCombinerPattern::REASSOC_XA_BY:
1281 OperandIndices = {1, 2, 1, 1, 2};
1282 break;
1283 case MachineCombinerPattern::REASSOC_XA_YB:
1284 OperandIndices = {2, 2, 2, 1, 1};
1285 break;
1286 default:
1287 llvm_unreachable("unexpected MachineCombinerPattern");
1288 }
1289}
1290
1291/// Attempt the reassociation transformation to reduce critical path length.
1292/// See the above comments before getMachineCombinerPatterns().
1293void TargetInstrInfo::reassociateOps(
1294 MachineInstr &Root, MachineInstr &Prev, unsigned Pattern,
1295 SmallVectorImpl<MachineInstr *> &InsInstrs,
1296 SmallVectorImpl<MachineInstr *> &DelInstrs,
1297 ArrayRef<unsigned> OperandIndices,
1298 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
1299 MachineFunction *MF = Root.getMF();
1300 MachineRegisterInfo &MRI = MF->getRegInfo();
1301 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1302 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1303 const TargetRegisterClass *RC = Root.getRegClassConstraint(OpIdx: 0, TII, TRI);
1304
1305 MachineOperand &OpA = Prev.getOperand(i: OperandIndices[1]);
1306 MachineOperand &OpB = Root.getOperand(i: OperandIndices[2]);
1307 MachineOperand &OpX = Prev.getOperand(i: OperandIndices[3]);
1308 MachineOperand &OpY = Root.getOperand(i: OperandIndices[4]);
1309 MachineOperand &OpC = Root.getOperand(i: 0);
1310
1311 Register RegA = OpA.getReg();
1312 Register RegB = OpB.getReg();
1313 Register RegX = OpX.getReg();
1314 Register RegY = OpY.getReg();
1315 Register RegC = OpC.getReg();
1316
1317 if (RegA.isVirtual())
1318 MRI.constrainRegClass(Reg: RegA, RC);
1319 if (RegB.isVirtual())
1320 MRI.constrainRegClass(Reg: RegB, RC);
1321 if (RegX.isVirtual())
1322 MRI.constrainRegClass(Reg: RegX, RC);
1323 if (RegY.isVirtual())
1324 MRI.constrainRegClass(Reg: RegY, RC);
1325 if (RegC.isVirtual())
1326 MRI.constrainRegClass(Reg: RegC, RC);
1327
1328 // Create a new virtual register for the result of (X op Y) instead of
1329 // recycling RegB because the MachineCombiner's computation of the critical
1330 // path requires a new register definition rather than an existing one.
1331 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
1332 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
1333
1334 auto [NewRootOpc, NewPrevOpc] = getReassociationOpcodes(Pattern, Root, Prev);
1335 bool KillA = OpA.isKill();
1336 bool KillX = OpX.isKill();
1337 bool KillY = OpY.isKill();
1338 bool KillNewVR = true;
1339
1340 auto [SwapRootOperands, SwapPrevOperands] = mustSwapOperands(Pattern);
1341
1342 if (SwapPrevOperands) {
1343 std::swap(a&: RegX, b&: RegY);
1344 std::swap(a&: KillX, b&: KillY);
1345 }
1346
1347 unsigned PrevFirstOpIdx, PrevSecondOpIdx;
1348 unsigned RootFirstOpIdx, RootSecondOpIdx;
1349 switch (Pattern) {
1350 case MachineCombinerPattern::REASSOC_AX_BY:
1351 PrevFirstOpIdx = OperandIndices[1];
1352 PrevSecondOpIdx = OperandIndices[3];
1353 RootFirstOpIdx = OperandIndices[2];
1354 RootSecondOpIdx = OperandIndices[4];
1355 break;
1356 case MachineCombinerPattern::REASSOC_AX_YB:
1357 PrevFirstOpIdx = OperandIndices[1];
1358 PrevSecondOpIdx = OperandIndices[3];
1359 RootFirstOpIdx = OperandIndices[4];
1360 RootSecondOpIdx = OperandIndices[2];
1361 break;
1362 case MachineCombinerPattern::REASSOC_XA_BY:
1363 PrevFirstOpIdx = OperandIndices[3];
1364 PrevSecondOpIdx = OperandIndices[1];
1365 RootFirstOpIdx = OperandIndices[2];
1366 RootSecondOpIdx = OperandIndices[4];
1367 break;
1368 case MachineCombinerPattern::REASSOC_XA_YB:
1369 PrevFirstOpIdx = OperandIndices[3];
1370 PrevSecondOpIdx = OperandIndices[1];
1371 RootFirstOpIdx = OperandIndices[4];
1372 RootSecondOpIdx = OperandIndices[2];
1373 break;
1374 default:
1375 llvm_unreachable("unexpected MachineCombinerPattern");
1376 }
1377
1378 // Basically BuildMI but doesn't add implicit operands by default.
1379 auto buildMINoImplicit = [](MachineFunction &MF, const MIMetadata &MIMD,
1380 const MCInstrDesc &MCID, Register DestReg) {
1381 return MachineInstrBuilder(
1382 MF, MF.CreateMachineInstr(MCID, DL: MIMD.getDL(), /*NoImpl=*/NoImplicit: true))
1383 .setPCSections(MIMD.getPCSections())
1384 .addReg(RegNo: DestReg, flags: RegState::Define);
1385 };
1386
1387 // Create new instructions for insertion.
1388 MachineInstrBuilder MIB1 =
1389 buildMINoImplicit(*MF, MIMetadata(Prev), TII->get(Opcode: NewPrevOpc), NewVR);
1390 for (const auto &MO : Prev.explicit_operands()) {
1391 unsigned Idx = MO.getOperandNo();
1392 // Skip the result operand we'd already added.
1393 if (Idx == 0)
1394 continue;
1395 if (Idx == PrevFirstOpIdx)
1396 MIB1.addReg(RegNo: RegX, flags: getKillRegState(B: KillX));
1397 else if (Idx == PrevSecondOpIdx)
1398 MIB1.addReg(RegNo: RegY, flags: getKillRegState(B: KillY));
1399 else
1400 MIB1.add(MO);
1401 }
1402 MIB1.copyImplicitOps(OtherMI: Prev);
1403
1404 if (SwapRootOperands) {
1405 std::swap(a&: RegA, b&: NewVR);
1406 std::swap(a&: KillA, b&: KillNewVR);
1407 }
1408
1409 MachineInstrBuilder MIB2 =
1410 buildMINoImplicit(*MF, MIMetadata(Root), TII->get(Opcode: NewRootOpc), RegC);
1411 for (const auto &MO : Root.explicit_operands()) {
1412 unsigned Idx = MO.getOperandNo();
1413 // Skip the result operand.
1414 if (Idx == 0)
1415 continue;
1416 if (Idx == RootFirstOpIdx)
1417 MIB2 = MIB2.addReg(RegNo: RegA, flags: getKillRegState(B: KillA));
1418 else if (Idx == RootSecondOpIdx)
1419 MIB2 = MIB2.addReg(RegNo: NewVR, flags: getKillRegState(B: KillNewVR));
1420 else
1421 MIB2 = MIB2.add(MO);
1422 }
1423 MIB2.copyImplicitOps(OtherMI: Root);
1424
1425 // Propagate FP flags from the original instructions.
1426 // But clear poison-generating flags because those may not be valid now.
1427 // TODO: There should be a helper function for copying only fast-math-flags.
1428 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
1429 MIB1->setFlags(IntersectedFlags);
1430 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1431 MIB1->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1432 MIB1->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1433
1434 MIB2->setFlags(IntersectedFlags);
1435 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
1436 MIB2->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
1437 MIB2->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
1438
1439 setSpecialOperandAttr(OldMI1&: Root, OldMI2&: Prev, NewMI1&: *MIB1, NewMI2&: *MIB2);
1440
1441 // Record new instructions for insertion and old instructions for deletion.
1442 InsInstrs.push_back(Elt: MIB1);
1443 InsInstrs.push_back(Elt: MIB2);
1444 DelInstrs.push_back(Elt: &Prev);
1445 DelInstrs.push_back(Elt: &Root);
1446
1447 // We transformed:
1448 // B = A op X (Prev)
1449 // C = B op Y (Root)
1450 // Into:
1451 // B = X op Y (MIB1)
1452 // C = A op B (MIB2)
1453 // C has the same value as before, B doesn't; as such, keep the debug number
1454 // of C but not of B.
1455 if (unsigned OldRootNum = Root.peekDebugInstrNum())
1456 MIB2.getInstr()->setDebugInstrNum(OldRootNum);
1457}
1458
1459void TargetInstrInfo::genAlternativeCodeSequence(
1460 MachineInstr &Root, unsigned Pattern,
1461 SmallVectorImpl<MachineInstr *> &InsInstrs,
1462 SmallVectorImpl<MachineInstr *> &DelInstrs,
1463 DenseMap<Register, unsigned> &InstIdxForVirtReg) const {
1464 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1465 MachineBasicBlock &MBB = *Root.getParent();
1466 MachineFunction &MF = *MBB.getParent();
1467 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1468
1469 switch (Pattern) {
1470 case MachineCombinerPattern::REASSOC_AX_BY:
1471 case MachineCombinerPattern::REASSOC_AX_YB:
1472 case MachineCombinerPattern::REASSOC_XA_BY:
1473 case MachineCombinerPattern::REASSOC_XA_YB: {
1474 // Select the previous instruction in the sequence based on the input
1475 // pattern.
1476 std::array<unsigned, 5> OperandIndices;
1477 getReassociateOperandIndices(Root, Pattern, OperandIndices);
1478 MachineInstr *Prev =
1479 MRI.getUniqueVRegDef(Reg: Root.getOperand(i: OperandIndices[0]).getReg());
1480
1481 // Don't reassociate if Prev and Root are in different blocks.
1482 if (Prev->getParent() != Root.getParent())
1483 return;
1484
1485 reassociateOps(Root, Prev&: *Prev, Pattern, InsInstrs, DelInstrs, OperandIndices,
1486 InstrIdxForVirtReg&: InstIdxForVirtReg);
1487 break;
1488 }
1489 case MachineCombinerPattern::ACC_CHAIN: {
1490 SmallVector<Register, 32> ChainRegs;
1491 getAccumulatorChain(CurrentInstr: &Root, Chain&: ChainRegs);
1492 unsigned int Depth = ChainRegs.size();
1493 assert(MaxAccumulatorWidth > 1 &&
1494 "Max accumulator width set to illegal value");
1495 unsigned int MaxWidth = Log2_32(Value: Depth) < MaxAccumulatorWidth
1496 ? Log2_32(Value: Depth)
1497 : MaxAccumulatorWidth;
1498
1499 // Walk down the chain and rewrite it as a tree.
1500 for (auto IndexedReg : llvm::enumerate(First: llvm::reverse(C&: ChainRegs))) {
1501 // No need to rewrite the first node, it is already perfect as it is.
1502 if (IndexedReg.index() == 0)
1503 continue;
1504
1505 MachineInstr *Instr = MRI.getUniqueVRegDef(Reg: IndexedReg.value());
1506 MachineInstrBuilder MIB;
1507 Register AccReg;
1508 if (IndexedReg.index() < MaxWidth) {
1509 // Now we need to create new instructions for the first row.
1510 AccReg = Instr->getOperand(i: 0).getReg();
1511 unsigned OpCode = getAccumulationStartOpcode(Opcode: Root.getOpcode());
1512
1513 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: OpCode), DestReg: AccReg)
1514 .addReg(RegNo: Instr->getOperand(i: 2).getReg(),
1515 flags: getKillRegState(B: Instr->getOperand(i: 2).isKill()))
1516 .addReg(RegNo: Instr->getOperand(i: 3).getReg(),
1517 flags: getKillRegState(B: Instr->getOperand(i: 3).isKill()));
1518 } else {
1519 // For the remaining cases, we need to use an output register of one of
1520 // the newly inserted instuctions as operand 1
1521 AccReg = Instr->getOperand(i: 0).getReg() == Root.getOperand(i: 0).getReg()
1522 ? MRI.createVirtualRegister(
1523 RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg()))
1524 : Instr->getOperand(i: 0).getReg();
1525 assert(IndexedReg.index() >= MaxWidth);
1526 auto AccumulatorInput =
1527 ChainRegs[Depth - (IndexedReg.index() - MaxWidth) - 1];
1528 MIB = BuildMI(MF, MIMD: MIMetadata(*Instr), MCID: TII->get(Opcode: Instr->getOpcode()),
1529 DestReg: AccReg)
1530 .addReg(RegNo: AccumulatorInput, flags: getKillRegState(B: true))
1531 .addReg(RegNo: Instr->getOperand(i: 2).getReg(),
1532 flags: getKillRegState(B: Instr->getOperand(i: 2).isKill()))
1533 .addReg(RegNo: Instr->getOperand(i: 3).getReg(),
1534 flags: getKillRegState(B: Instr->getOperand(i: 3).isKill()));
1535 }
1536
1537 MIB->setFlags(Instr->getFlags());
1538 InstIdxForVirtReg.insert(KV: std::make_pair(x&: AccReg, y: InsInstrs.size()));
1539 InsInstrs.push_back(Elt: MIB);
1540 DelInstrs.push_back(Elt: Instr);
1541 }
1542
1543 SmallVector<Register, 8> RegistersToReduce;
1544 for (unsigned i = (InsInstrs.size() - MaxWidth); i < InsInstrs.size();
1545 ++i) {
1546 auto Reg = InsInstrs[i]->getOperand(i: 0).getReg();
1547 RegistersToReduce.push_back(Elt: Reg);
1548 }
1549
1550 while (RegistersToReduce.size() > 1)
1551 reduceAccumulatorTree(RegistersToReduce, InsInstrs, MF, Root, MRI,
1552 InstrIdxForVirtReg&: InstIdxForVirtReg, ResultReg: Root.getOperand(i: 0).getReg());
1553
1554 break;
1555 }
1556 }
1557}
1558
1559MachineTraceStrategy TargetInstrInfo::getMachineCombinerTraceStrategy() const {
1560 return MachineTraceStrategy::TS_MinInstrCount;
1561}
1562
1563bool TargetInstrInfo::isReallyTriviallyReMaterializable(
1564 const MachineInstr &MI) const {
1565 const MachineFunction &MF = *MI.getMF();
1566 const MachineRegisterInfo &MRI = MF.getRegInfo();
1567
1568 // Remat clients assume operand 0 is the defined register.
1569 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
1570 return false;
1571 Register DefReg = MI.getOperand(i: 0).getReg();
1572
1573 // A sub-register definition can only be rematerialized if the instruction
1574 // doesn't read the other parts of the register. Otherwise it is really a
1575 // read-modify-write operation on the full virtual register which cannot be
1576 // moved safely.
1577 if (DefReg.isVirtual() && MI.getOperand(i: 0).getSubReg() &&
1578 MI.readsVirtualRegister(Reg: DefReg))
1579 return false;
1580
1581 // A load from a fixed stack slot can be rematerialized. This may be
1582 // redundant with subsequent checks, but it's target-independent,
1583 // simple, and a common case.
1584 int FrameIdx = 0;
1585 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
1586 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
1587 return true;
1588
1589 // Avoid instructions obviously unsafe for remat.
1590 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
1591 MI.hasUnmodeledSideEffects())
1592 return false;
1593
1594 // Don't remat inline asm. We have no idea how expensive it is
1595 // even if it's side effect free.
1596 if (MI.isInlineAsm())
1597 return false;
1598
1599 // Avoid instructions which load from potentially varying memory.
1600 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
1601 return false;
1602
1603 // If any of the registers accessed are non-constant, conservatively assume
1604 // the instruction is not rematerializable.
1605 for (const MachineOperand &MO : MI.operands()) {
1606 if (!MO.isReg()) continue;
1607 Register Reg = MO.getReg();
1608 if (Reg == 0)
1609 continue;
1610
1611 // Check for a well-behaved physical register.
1612 if (Reg.isPhysical()) {
1613 if (MO.isUse()) {
1614 // If the physreg has no defs anywhere, it's just an ambient register
1615 // and we can freely move its uses. Alternatively, if it's allocatable,
1616 // it could get allocated to something with a def during allocation.
1617 if (!MRI.isConstantPhysReg(PhysReg: Reg))
1618 return false;
1619 } else {
1620 // A physreg def. We can't remat it.
1621 return false;
1622 }
1623 continue;
1624 }
1625
1626 // Only allow one virtual-register def. There may be multiple defs of the
1627 // same virtual register, though.
1628 if (MO.isDef() && Reg != DefReg)
1629 return false;
1630
1631 // Don't allow any virtual-register uses. Rematting an instruction with
1632 // virtual register uses would length the live ranges of the uses, which
1633 // is not necessarily a good idea, certainly not "trivial".
1634 if (MO.isUse())
1635 return false;
1636 }
1637
1638 // Everything checked out.
1639 return true;
1640}
1641
1642int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
1643 const MachineFunction *MF = MI.getMF();
1644 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
1645 bool StackGrowsDown =
1646 TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
1647
1648 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
1649 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
1650
1651 if (!isFrameInstr(I: MI))
1652 return 0;
1653
1654 int SPAdj = TFI->alignSPAdjust(SPAdj: getFrameSize(I: MI));
1655
1656 if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
1657 (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
1658 SPAdj = -SPAdj;
1659
1660 return SPAdj;
1661}
1662
1663/// isSchedulingBoundary - Test if the given instruction should be
1664/// considered a scheduling boundary. This primarily includes labels
1665/// and terminators.
1666bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1667 const MachineBasicBlock *MBB,
1668 const MachineFunction &MF) const {
1669 // Terminators and labels can't be scheduled around.
1670 if (MI.isTerminator() || MI.isPosition())
1671 return true;
1672
1673 // INLINEASM_BR can jump to another block
1674 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
1675 return true;
1676
1677 // Don't attempt to schedule around any instruction that defines
1678 // a stack-oriented pointer, as it's unlikely to be profitable. This
1679 // saves compile time, because it doesn't require every single
1680 // stack slot reference to depend on the instruction that does the
1681 // modification.
1682 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
1683 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1684 return MI.modifiesRegister(Reg: TLI.getStackPointerRegisterToSaveRestore(), TRI);
1685}
1686
1687// Provide a global flag for disabling the PreRA hazard recognizer that targets
1688// may choose to honor.
1689bool TargetInstrInfo::usePreRAHazardRecognizer() const {
1690 return !DisableHazardRecognizer;
1691}
1692
1693// Default implementation of CreateTargetRAHazardRecognizer.
1694ScheduleHazardRecognizer *TargetInstrInfo::
1695CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
1696 const ScheduleDAG *DAG) const {
1697 // Dummy hazard recognizer allows all instructions to issue.
1698 return new ScheduleHazardRecognizer();
1699}
1700
1701// Default implementation of CreateTargetMIHazardRecognizer.
1702ScheduleHazardRecognizer *TargetInstrInfo::CreateTargetMIHazardRecognizer(
1703 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
1704 return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler");
1705}
1706
1707// Default implementation of CreateTargetPostRAHazardRecognizer.
1708ScheduleHazardRecognizer *TargetInstrInfo::
1709CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1710 const ScheduleDAG *DAG) const {
1711 return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1712}
1713
1714// Default implementation of getMemOperandWithOffset.
1715bool TargetInstrInfo::getMemOperandWithOffset(
1716 const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
1717 bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
1718 SmallVector<const MachineOperand *, 4> BaseOps;
1719 LocationSize Width = LocationSize::precise(Value: 0);
1720 if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
1721 Width, TRI) ||
1722 BaseOps.size() != 1)
1723 return false;
1724 BaseOp = BaseOps.front();
1725 return true;
1726}
1727
1728//===----------------------------------------------------------------------===//
1729// SelectionDAG latency interface.
1730//===----------------------------------------------------------------------===//
1731
1732std::optional<unsigned>
1733TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1734 SDNode *DefNode, unsigned DefIdx,
1735 SDNode *UseNode, unsigned UseIdx) const {
1736 if (!ItinData || ItinData->isEmpty())
1737 return std::nullopt;
1738
1739 if (!DefNode->isMachineOpcode())
1740 return std::nullopt;
1741
1742 unsigned DefClass = get(Opcode: DefNode->getMachineOpcode()).getSchedClass();
1743 if (!UseNode->isMachineOpcode())
1744 return ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1745 unsigned UseClass = get(Opcode: UseNode->getMachineOpcode()).getSchedClass();
1746 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1747}
1748
1749unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1750 SDNode *N) const {
1751 if (!ItinData || ItinData->isEmpty())
1752 return 1;
1753
1754 if (!N->isMachineOpcode())
1755 return 1;
1756
1757 return ItinData->getStageLatency(ItinClassIndx: get(Opcode: N->getMachineOpcode()).getSchedClass());
1758}
1759
1760//===----------------------------------------------------------------------===//
1761// MachineInstr latency interface.
1762//===----------------------------------------------------------------------===//
1763
1764unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
1765 const MachineInstr &MI) const {
1766 if (!ItinData || ItinData->isEmpty())
1767 return 1;
1768
1769 unsigned Class = MI.getDesc().getSchedClass();
1770 int UOps = ItinData->Itineraries[Class].NumMicroOps;
1771 if (UOps >= 0)
1772 return UOps;
1773
1774 // The # of u-ops is dynamically determined. The specific target should
1775 // override this function to return the right number.
1776 return 1;
1777}
1778
1779/// Return the default expected latency for a def based on it's opcode.
1780unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel,
1781 const MachineInstr &DefMI) const {
1782 if (DefMI.isTransient())
1783 return 0;
1784 if (DefMI.mayLoad())
1785 return SchedModel.LoadLatency;
1786 if (isHighLatencyDef(opc: DefMI.getOpcode()))
1787 return SchedModel.HighLatency;
1788 return 1;
1789}
1790
1791unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const {
1792 return 0;
1793}
1794
1795unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1796 const MachineInstr &MI,
1797 unsigned *PredCost) const {
1798 // Default to one cycle for no itinerary. However, an "empty" itinerary may
1799 // still have a MinLatency property, which getStageLatency checks.
1800 if (!ItinData)
1801 return MI.mayLoad() ? 2 : 1;
1802
1803 return ItinData->getStageLatency(ItinClassIndx: MI.getDesc().getSchedClass());
1804}
1805
1806bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
1807 const MachineInstr &DefMI,
1808 unsigned DefIdx) const {
1809 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1810 if (!ItinData || ItinData->isEmpty())
1811 return false;
1812
1813 unsigned DefClass = DefMI.getDesc().getSchedClass();
1814 std::optional<unsigned> DefCycle =
1815 ItinData->getOperandCycle(ItinClassIndx: DefClass, OperandIdx: DefIdx);
1816 return DefCycle && DefCycle <= 1U;
1817}
1818
1819bool TargetInstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
1820 // TODO: We don't split functions where a section attribute has been set
1821 // since the split part may not be placed in a contiguous region. It may also
1822 // be more beneficial to augment the linker to ensure contiguous layout of
1823 // split functions within the same section as specified by the attribute.
1824 if (MF.getFunction().hasSection())
1825 return false;
1826
1827 // We don't want to proceed further for cold functions
1828 // or functions of unknown hotness. Lukewarm functions have no prefix.
1829 std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
1830 if (SectionPrefix &&
1831 (*SectionPrefix == "unlikely" || *SectionPrefix == "unknown")) {
1832 return false;
1833 }
1834
1835 return true;
1836}
1837
1838std::optional<ParamLoadedValue>
1839TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
1840 Register Reg) const {
1841 const MachineFunction *MF = MI.getMF();
1842 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1843 DIExpression *Expr = DIExpression::get(Context&: MF->getFunction().getContext(), Elements: {});
1844 int64_t Offset;
1845 bool OffsetIsScalable;
1846
1847 // To simplify the sub-register handling, verify that we only need to
1848 // consider physical registers.
1849 assert(MF->getProperties().hasNoVRegs());
1850
1851 if (auto DestSrc = isCopyInstr(MI)) {
1852 Register DestReg = DestSrc->Destination->getReg();
1853
1854 // If the copy destination is the forwarding reg, describe the forwarding
1855 // reg using the copy source as the backup location. Example:
1856 //
1857 // x0 = MOV x7
1858 // call callee(x0) ; x0 described as x7
1859 if (Reg == DestReg)
1860 return ParamLoadedValue(*DestSrc->Source, Expr);
1861
1862 // If the target's hook couldn't describe this copy, give up.
1863 return std::nullopt;
1864 } else if (auto RegImm = isAddImmediate(MI, Reg)) {
1865 Register SrcReg = RegImm->Reg;
1866 Offset = RegImm->Imm;
1867 Expr = DIExpression::prepend(Expr, Flags: DIExpression::ApplyOffset, Offset);
1868 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
1869 } else if (MI.hasOneMemOperand()) {
1870 // Only describe memory which provably does not escape the function. As
1871 // described in llvm.org/PR43343, escaped memory may be clobbered by the
1872 // callee (or by another thread).
1873 const auto &TII = MF->getSubtarget().getInstrInfo();
1874 const MachineFrameInfo &MFI = MF->getFrameInfo();
1875 const MachineMemOperand *MMO = MI.memoperands()[0];
1876 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1877
1878 // If the address points to "special" memory (e.g. a spill slot), it's
1879 // sufficient to check that it isn't aliased by any high-level IR value.
1880 if (!PSV || PSV->mayAlias(&MFI))
1881 return std::nullopt;
1882
1883 const MachineOperand *BaseOp;
1884 if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
1885 TRI))
1886 return std::nullopt;
1887
1888 // FIXME: Scalable offsets are not yet handled in the offset code below.
1889 if (OffsetIsScalable)
1890 return std::nullopt;
1891
1892 // TODO: Can currently only handle mem instructions with a single define.
1893 // An example from the x86 target:
1894 // ...
1895 // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx
1896 // ...
1897 //
1898 if (MI.getNumExplicitDefs() != 1)
1899 return std::nullopt;
1900
1901 // TODO: In what way do we need to take Reg into consideration here?
1902
1903 SmallVector<uint64_t, 8> Ops;
1904 DIExpression::appendOffset(Ops, Offset);
1905 Ops.push_back(Elt: dwarf::DW_OP_deref_size);
1906 Ops.push_back(Elt: MMO->getSize().hasValue() ? MMO->getSize().getValue()
1907 : ~UINT64_C(0));
1908 Expr = DIExpression::prependOpcodes(Expr, Ops);
1909 return ParamLoadedValue(*BaseOp, Expr);
1910 }
1911
1912 return std::nullopt;
1913}
1914
1915// Get the call frame size just before MI.
1916unsigned TargetInstrInfo::getCallFrameSizeAt(MachineInstr &MI) const {
1917 // Search backwards from MI for the most recent call frame instruction.
1918 MachineBasicBlock *MBB = MI.getParent();
1919 for (auto &AdjI : reverse(C: make_range(x: MBB->instr_begin(), y: MI.getIterator()))) {
1920 if (AdjI.getOpcode() == getCallFrameSetupOpcode())
1921 return getFrameTotalSize(I: AdjI);
1922 if (AdjI.getOpcode() == getCallFrameDestroyOpcode())
1923 return 0;
1924 }
1925
1926 // If none was found, use the call frame size from the start of the basic
1927 // block.
1928 return MBB->getCallFrameSize();
1929}
1930
1931/// Both DefMI and UseMI must be valid. By default, call directly to the
1932/// itinerary. This may be overriden by the target.
1933std::optional<unsigned> TargetInstrInfo::getOperandLatency(
1934 const InstrItineraryData *ItinData, const MachineInstr &DefMI,
1935 unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
1936 unsigned DefClass = DefMI.getDesc().getSchedClass();
1937 unsigned UseClass = UseMI.getDesc().getSchedClass();
1938 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1939}
1940
1941bool TargetInstrInfo::getRegSequenceInputs(
1942 const MachineInstr &MI, unsigned DefIdx,
1943 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1944 assert((MI.isRegSequence() ||
1945 MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1946
1947 if (!MI.isRegSequence())
1948 return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1949
1950 // We are looking at:
1951 // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
1952 assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
1953 for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
1954 OpIdx += 2) {
1955 const MachineOperand &MOReg = MI.getOperand(i: OpIdx);
1956 if (MOReg.isUndef())
1957 continue;
1958 const MachineOperand &MOSubIdx = MI.getOperand(i: OpIdx + 1);
1959 assert(MOSubIdx.isImm() &&
1960 "One of the subindex of the reg_sequence is not an immediate");
1961 // Record Reg:SubReg, SubIdx.
1962 InputRegs.push_back(Elt: RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
1963 (unsigned)MOSubIdx.getImm()));
1964 }
1965 return true;
1966}
1967
1968bool TargetInstrInfo::getExtractSubregInputs(
1969 const MachineInstr &MI, unsigned DefIdx,
1970 RegSubRegPairAndIdx &InputReg) const {
1971 assert((MI.isExtractSubreg() ||
1972 MI.isExtractSubregLike()) && "Instruction do not have the proper type");
1973
1974 if (!MI.isExtractSubreg())
1975 return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
1976
1977 // We are looking at:
1978 // Def = EXTRACT_SUBREG v0.sub1, sub0.
1979 assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
1980 const MachineOperand &MOReg = MI.getOperand(i: 1);
1981 if (MOReg.isUndef())
1982 return false;
1983 const MachineOperand &MOSubIdx = MI.getOperand(i: 2);
1984 assert(MOSubIdx.isImm() &&
1985 "The subindex of the extract_subreg is not an immediate");
1986
1987 InputReg.Reg = MOReg.getReg();
1988 InputReg.SubReg = MOReg.getSubReg();
1989 InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
1990 return true;
1991}
1992
1993bool TargetInstrInfo::getInsertSubregInputs(
1994 const MachineInstr &MI, unsigned DefIdx,
1995 RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
1996 assert((MI.isInsertSubreg() ||
1997 MI.isInsertSubregLike()) && "Instruction do not have the proper type");
1998
1999 if (!MI.isInsertSubreg())
2000 return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
2001
2002 // We are looking at:
2003 // Def = INSERT_SEQUENCE v0, v1, sub0.
2004 assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
2005 const MachineOperand &MOBaseReg = MI.getOperand(i: 1);
2006 const MachineOperand &MOInsertedReg = MI.getOperand(i: 2);
2007 if (MOInsertedReg.isUndef())
2008 return false;
2009 const MachineOperand &MOSubIdx = MI.getOperand(i: 3);
2010 assert(MOSubIdx.isImm() &&
2011 "One of the subindex of the reg_sequence is not an immediate");
2012 BaseReg.Reg = MOBaseReg.getReg();
2013 BaseReg.SubReg = MOBaseReg.getSubReg();
2014
2015 InsertedReg.Reg = MOInsertedReg.getReg();
2016 InsertedReg.SubReg = MOInsertedReg.getSubReg();
2017 InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
2018 return true;
2019}
2020
2021// Returns a MIRPrinter comment for this machine operand.
2022std::string TargetInstrInfo::createMIROperandComment(
2023 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2024 const TargetRegisterInfo *TRI) const {
2025
2026 if (!MI.isInlineAsm())
2027 return "";
2028
2029 std::string Flags;
2030 raw_string_ostream OS(Flags);
2031
2032 if (OpIdx == InlineAsm::MIOp_ExtraInfo) {
2033 // Print HasSideEffects, MayLoad, MayStore, IsAlignStack
2034 unsigned ExtraInfo = Op.getImm();
2035 bool First = true;
2036 for (StringRef Info : InlineAsm::getExtraInfoNames(ExtraInfo)) {
2037 if (!First)
2038 OS << " ";
2039 First = false;
2040 OS << Info;
2041 }
2042
2043 return Flags;
2044 }
2045
2046 int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx);
2047 if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx)
2048 return "";
2049
2050 assert(Op.isImm() && "Expected flag operand to be an immediate");
2051 // Pretty print the inline asm operand descriptor.
2052 unsigned Flag = Op.getImm();
2053 const InlineAsm::Flag F(Flag);
2054 OS << F.getKindName();
2055
2056 unsigned RCID;
2057 if (!F.isImmKind() && !F.isMemKind() && F.hasRegClassConstraint(RC&: RCID)) {
2058 if (TRI) {
2059 OS << ':' << TRI->getRegClassName(Class: TRI->getRegClass(i: RCID));
2060 } else
2061 OS << ":RC" << RCID;
2062 }
2063
2064 if (F.isMemKind()) {
2065 InlineAsm::ConstraintCode MCID = F.getMemoryConstraintID();
2066 OS << ":" << InlineAsm::getMemConstraintName(C: MCID);
2067 }
2068
2069 unsigned TiedTo;
2070 if (F.isUseOperandTiedToDef(Idx&: TiedTo))
2071 OS << " tiedto:$" << TiedTo;
2072
2073 if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isRegUseKind()) &&
2074 F.getRegMayBeFolded())
2075 OS << " foldable";
2076
2077 return Flags;
2078}
2079
2080TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() = default;
2081
2082void TargetInstrInfo::mergeOutliningCandidateAttributes(
2083 Function &F, std::vector<outliner::Candidate> &Candidates) const {
2084 // Include target features from an arbitrary candidate for the outlined
2085 // function. This makes sure the outlined function knows what kinds of
2086 // instructions are going into it. This is fine, since all parent functions
2087 // must necessarily support the instructions that are in the outlined region.
2088 outliner::Candidate &FirstCand = Candidates.front();
2089 const Function &ParentFn = FirstCand.getMF()->getFunction();
2090 if (ParentFn.hasFnAttribute(Kind: "target-features"))
2091 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-features"));
2092 if (ParentFn.hasFnAttribute(Kind: "target-cpu"))
2093 F.addFnAttr(Attr: ParentFn.getFnAttribute(Kind: "target-cpu"));
2094
2095 // Set nounwind, so we don't generate eh_frame.
2096 if (llvm::all_of(Range&: Candidates, P: [](const outliner::Candidate &C) {
2097 return C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
2098 }))
2099 F.addFnAttr(Kind: Attribute::NoUnwind);
2100}
2101
2102outliner::InstrType
2103TargetInstrInfo::getOutliningType(const MachineModuleInfo &MMI,
2104 MachineBasicBlock::iterator &MIT,
2105 unsigned Flags) const {
2106 MachineInstr &MI = *MIT;
2107
2108 // NOTE: MI.isMetaInstruction() will match CFI_INSTRUCTION, but some targets
2109 // have support for outlining those. Special-case that here.
2110 if (MI.isCFIInstruction())
2111 // Just go right to the target implementation.
2112 return getOutliningTypeImpl(MMI, MIT, Flags);
2113
2114 // Be conservative about inline assembly.
2115 if (MI.isInlineAsm())
2116 return outliner::InstrType::Illegal;
2117
2118 // Labels generally can't safely be outlined.
2119 if (MI.isLabel())
2120 return outliner::InstrType::Illegal;
2121
2122 // Don't let debug instructions impact analysis.
2123 if (MI.isDebugInstr())
2124 return outliner::InstrType::Invisible;
2125
2126 // Some other special cases.
2127 switch (MI.getOpcode()) {
2128 case TargetOpcode::IMPLICIT_DEF:
2129 case TargetOpcode::KILL:
2130 case TargetOpcode::LIFETIME_START:
2131 case TargetOpcode::LIFETIME_END:
2132 return outliner::InstrType::Invisible;
2133 default:
2134 break;
2135 }
2136
2137 // Is this a terminator for a basic block?
2138 if (MI.isTerminator()) {
2139 // If this is a branch to another block, we can't outline it.
2140 if (!MI.getParent()->succ_empty())
2141 return outliner::InstrType::Illegal;
2142
2143 // Don't outline if the branch is not unconditional.
2144 if (isPredicated(MI))
2145 return outliner::InstrType::Illegal;
2146 }
2147
2148 // Make sure none of the operands of this instruction do anything that
2149 // might break if they're moved outside their current function.
2150 // This includes MachineBasicBlock references, BlockAddressses,
2151 // Constant pool indices and jump table indices.
2152 //
2153 // A quick note on MO_TargetIndex:
2154 // This doesn't seem to be used in any of the architectures that the
2155 // MachineOutliner supports, but it was still filtered out in all of them.
2156 // There was one exception (RISC-V), but MO_TargetIndex also isn't used there.
2157 // As such, this check is removed both here and in the target-specific
2158 // implementations. Instead, we assert to make sure this doesn't
2159 // catch anyone off-guard somewhere down the line.
2160 for (const MachineOperand &MOP : MI.operands()) {
2161 // If you hit this assertion, please remove it and adjust
2162 // `getOutliningTypeImpl` for your target appropriately if necessary.
2163 // Adding the assertion back to other supported architectures
2164 // would be nice too :)
2165 assert(!MOP.isTargetIndex() && "This isn't used quite yet!");
2166
2167 // CFI instructions should already have been filtered out at this point.
2168 assert(!MOP.isCFIIndex() && "CFI instructions handled elsewhere!");
2169
2170 // PrologEpilogInserter should've already run at this point.
2171 assert(!MOP.isFI() && "FrameIndex instructions should be gone by now!");
2172
2173 if (MOP.isMBB() || MOP.isBlockAddress() || MOP.isCPI() || MOP.isJTI())
2174 return outliner::InstrType::Illegal;
2175 }
2176
2177 // If we don't know, delegate to the target-specific hook.
2178 return getOutliningTypeImpl(MMI, MIT, Flags);
2179}
2180
2181bool TargetInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
2182 unsigned &Flags) const {
2183 // Some instrumentations create special TargetOpcode at the start which
2184 // expands to special code sequences which must be present.
2185 auto First = MBB.getFirstNonDebugInstr();
2186 if (First == MBB.end())
2187 return true;
2188
2189 if (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
2190 First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)
2191 return false;
2192
2193 // Some instrumentations create special pseudo-instructions at or just before
2194 // the end that must be present.
2195 auto Last = MBB.getLastNonDebugInstr();
2196 if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET ||
2197 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2198 return false;
2199
2200 if (Last != First && Last->isReturn()) {
2201 --Last;
2202 if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT ||
2203 Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
2204 return false;
2205 }
2206 return true;
2207}
2208
2209bool TargetInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
2210 return MI->isCall() || MI->hasUnmodeledSideEffects() ||
2211 (MI->hasOrderedMemoryRef() && !MI->isDereferenceableInvariantLoad());
2212}
2213