1//===-- SIPreEmitPeephole.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass performs the peephole optimizations before code emission.
11///
12/// Additionally, this pass also unpacks packed instructions (V_PK_MUL_F32/F16,
13/// V_PK_ADD_F32/F16, V_PK_FMA_F32) adjacent to MFMAs such that they can be
14/// co-issued. This helps with overlapping MFMA and certain vector instructions
15/// in machine schedules and is expected to improve performance. Only those
16/// packed instructions are unpacked that are overlapped by the MFMA latency.
17/// Rest should remain untouched.
18/// TODO: Add support for F16 packed instructions
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "GCNSubtarget.h"
23#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/CodeGen/MachineDominators.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineLoopInfo.h"
29#include "llvm/CodeGen/MachinePostDominators.h"
30#include "llvm/CodeGen/TargetSchedule.h"
31#include "llvm/Support/BranchProbability.h"
32using namespace llvm;
33
34#define DEBUG_TYPE "si-pre-emit-peephole"
35
36STATISTIC(NumModeWritesRemoved,
37 "Number of redundant mode register writes removed");
38
39namespace {
40
41/// The state of one independent field of the MODE register, as tracked by
42/// removeRedundantModeWrites.
43struct ModeFieldState {
44 std::optional<int64_t> Value;
45 std::optional<int64_t> ValueBeforePendingWrite;
46 MachineInstr *PendingWrite = nullptr;
47
48 bool isTracked() const { return PendingWrite || Value; }
49};
50
51class SIPreEmitPeephole {
52private:
53 const SIInstrInfo *TII = nullptr;
54 const SIRegisterInfo *TRI = nullptr;
55 MachineLoopInfo *MLI = nullptr;
56
57 bool optimizeVccBranch(MachineInstr &MI) const;
58 void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
59 MachineBasicBlock *To) const;
60 bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
61 bool getBlockDestinations(MachineBasicBlock &SrcMBB,
62 MachineBasicBlock *&TrueMBB,
63 MachineBasicBlock *&FalseMBB,
64 SmallVectorImpl<MachineOperand> &Cond);
65 bool mustRetainExeczBranch(const MachineInstr &Branch,
66 const MachineBasicBlock &From,
67 const MachineBasicBlock &To) const;
68 bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
69 bool removeRedundantModeWrites(MachineBasicBlock &SrcMBB) const;
70 // Creates a list of packed instructions following an MFMA that are suitable
71 // for unpacking.
72 void collectUnpackingCandidates(MachineInstr &BeginMI,
73 SetVector<MachineInstr *> &InstrsToUnpack,
74 uint16_t NumMFMACycles);
75 // v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[2:3] op_sel:[1,1,1]
76 // op_sel_hi:[0,0,0]
77 // ==>
78 // v_fma_f32 v0, v1, v3, v3
79 // v_fma_f32 v1, v0, v2, v2
80 // Here, we have overwritten v0 before we use it. This function checks if
81 // unpacking can lead to such a situation.
82 bool canUnpackingClobberRegister(const MachineInstr &MI);
83 // Unpack and insert F32 packed instructions, such as V_PK_MUL, V_PK_ADD, and
84 // V_PK_FMA. Currently, only V_PK_MUL, V_PK_ADD, V_PK_FMA are supported for
85 // this transformation.
86 void performF32Unpacking(MachineInstr &I);
87 // Select corresponding unpacked instruction
88 uint32_t mapToUnpackedOpcode(MachineInstr &I);
89 // Creates the unpacked instruction to be inserted. Adds source modifiers to
90 // the unpacked instructions based on the source modifiers in the packed
91 // instruction.
92 MachineInstrBuilder createUnpackedMI(MachineInstr &I, uint32_t UnpackedOpcode,
93 bool IsHiBits);
94 // Process operands/source modifiers from packed instructions and insert the
95 // appropriate source modifers and operands into the unpacked instructions.
96 void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
97 bool IsHiBits, const MachineOperand &SrcMO);
98
99public:
100 bool run(MachineFunction &MF, MachineLoopInfo *MLI);
101};
102
103class SIPreEmitPeepholeLegacy : public MachineFunctionPass {
104public:
105 static char ID;
106
107 SIPreEmitPeepholeLegacy() : MachineFunctionPass(ID) {}
108
109 void getAnalysisUsage(AnalysisUsage &AU) const override {
110 AU.addUsedIfAvailable<MachineLoopInfoWrapperPass>();
111 AU.addPreserved<MachineLoopInfoWrapperPass>();
112 MachineFunctionPass::getAnalysisUsage(AU);
113 }
114
115 bool runOnMachineFunction(MachineFunction &MF) override {
116 auto *MLIWrapper = getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
117 MachineLoopInfo *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
118 return SIPreEmitPeephole().run(MF, MLI);
119 }
120};
121
122} // End anonymous namespace.
123
124INITIALIZE_PASS(SIPreEmitPeepholeLegacy, DEBUG_TYPE,
125 "SI peephole optimizations", false, false)
126
127char SIPreEmitPeepholeLegacy::ID = 0;
128
129char &llvm::SIPreEmitPeepholeID = SIPreEmitPeepholeLegacy::ID;
130
131void SIPreEmitPeephole::updateMLIBeforeRemovingEdge(
132 MachineBasicBlock *From, MachineBasicBlock *To) const {
133 if (!MLI)
134 return;
135
136 // Only handle back-edges: To must be a loop header with From inside the loop.
137 MachineLoop *Loop = MLI->getLoopFor(BB: To);
138 if (!Loop || Loop->getHeader() != To || !Loop->contains(BB: From))
139 return;
140
141 // Count back-edges
142 unsigned BackEdgeCount = 0;
143 for (MachineBasicBlock *Pred : To->predecessors()) {
144 if (Loop->contains(BB: Pred))
145 BackEdgeCount++;
146 }
147
148 if (BackEdgeCount > 1)
149 return;
150
151 MachineLoop *ParentLoop = Loop->getParentLoop();
152
153 // Re-map blocks directly owned by this loop to the parent.
154 for (MachineBasicBlock *BB : Loop->blocks()) {
155 if (MLI->getLoopFor(BB) == Loop)
156 MLI->changeLoopFor(BB, L: ParentLoop);
157 }
158
159 // Reparent all child loops.
160 while (!Loop->isInnermost()) {
161 MachineLoop *Child = Loop->removeChildLoop(I: std::prev(x: Loop->end()));
162 if (ParentLoop)
163 ParentLoop->addChildLoop(NewChild: Child);
164 else
165 MLI->addTopLevelLoop(New: Child);
166 }
167
168 if (ParentLoop)
169 ParentLoop->removeChildLoop(Child: Loop);
170 else
171 MLI->removeLoop(I: llvm::find(Range&: *MLI, Val: Loop));
172
173 MLI->destroy(L: Loop);
174}
175
176bool SIPreEmitPeephole::optimizeVccBranch(MachineInstr &MI) const {
177 // Match:
178 // sreg = -1 or 0
179 // vcc = S_AND_B64 exec, sreg or S_ANDN2_B64 exec, sreg
180 // S_CBRANCH_VCC[N]Z
181 // =>
182 // S_CBRANCH_EXEC[N]Z
183 // We end up with this pattern sometimes after basic block placement.
184 // It happens while combining a block which assigns -1 or 0 to a saved mask
185 // and another block which consumes that saved mask and then a branch.
186 //
187 // While searching this also performs the following substitution:
188 // vcc = V_CMP
189 // vcc = S_AND exec, vcc
190 // S_CBRANCH_VCC[N]Z
191 // =>
192 // vcc = V_CMP
193 // S_CBRANCH_VCC[N]Z
194
195 bool Changed = false;
196 MachineBasicBlock &MBB = *MI.getParent();
197 const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
198 const bool IsWave32 = ST.isWave32();
199 const unsigned CondReg = TRI->getVCC();
200 const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
201 const unsigned And = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
202 const unsigned AndN2 = IsWave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64;
203 const unsigned Mov = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
204
205 MachineBasicBlock::reverse_iterator A = MI.getReverseIterator(),
206 E = MBB.rend();
207 bool ReadsCond = false;
208 unsigned Threshold = 5;
209 for (++A; A != E; ++A) {
210 if (!--Threshold)
211 return false;
212 if (A->modifiesRegister(Reg: ExecReg, TRI))
213 return false;
214 if (A->modifiesRegister(Reg: CondReg, TRI)) {
215 if (!A->definesRegister(Reg: CondReg, TRI) ||
216 (A->getOpcode() != And && A->getOpcode() != AndN2))
217 return false;
218 break;
219 }
220 ReadsCond |= A->readsRegister(Reg: CondReg, TRI);
221 }
222 if (A == E)
223 return false;
224
225 MachineOperand &Op1 = A->getOperand(i: 1);
226 MachineOperand &Op2 = A->getOperand(i: 2);
227 if ((!Op1.isReg() || Op1.getReg() != ExecReg) && Op2.isReg() &&
228 Op2.getReg() == ExecReg) {
229 TII->commuteInstruction(MI&: *A);
230 Changed = true;
231 }
232 if (!Op1.isReg() || Op1.getReg() != ExecReg)
233 return Changed;
234 if (Op2.isImm() && !(Op2.getImm() == -1 || Op2.getImm() == 0))
235 return Changed;
236
237 int64_t MaskValue = 0;
238 Register SReg;
239 if (Op2.isReg()) {
240 SReg = Op2.getReg();
241 auto M = std::next(x: A);
242 bool ReadsSreg = false;
243 bool ModifiesExec = false;
244 for (; M != E; ++M) {
245 if (M->definesRegister(Reg: SReg, TRI))
246 break;
247 if (M->modifiesRegister(Reg: SReg, TRI))
248 return Changed;
249 ReadsSreg |= M->readsRegister(Reg: SReg, TRI);
250 ModifiesExec |= M->modifiesRegister(Reg: ExecReg, TRI);
251 }
252 if (M == E)
253 return Changed;
254 // If SReg is VCC and SReg definition is a VALU comparison.
255 // This means S_AND with EXEC is not required, unless
256 // the implicit def of SCC is alive.
257 // Erase the S_AND and return.
258 // Note: isVOPC is used instead of isCompare to catch V_CMP_CLASS
259 if (A->getOpcode() == And && SReg == CondReg && !ModifiesExec &&
260 TII->isVOPC(MI: *M) && A->allImplicitDefsAreDead()) {
261 A->eraseFromParent();
262 return true;
263 }
264
265 if (!M->isMoveImmediate() || !M->getOperand(i: 1).isImm() ||
266 (M->getOperand(i: 1).getImm() != -1 && M->getOperand(i: 1).getImm() != 0))
267 return Changed;
268 MaskValue = M->getOperand(i: 1).getImm();
269 // First if sreg is only used in the AND instruction fold the immediate
270 // into the AND.
271 if (!ReadsSreg && Op2.isKill()) {
272 A->getOperand(i: 2).ChangeToImmediate(ImmVal: MaskValue);
273 M->eraseFromParent();
274 }
275 } else if (Op2.isImm()) {
276 MaskValue = Op2.getImm();
277 } else {
278 llvm_unreachable("Op2 must be register or immediate");
279 }
280
281 // Invert mask for s_andn2
282 assert(MaskValue == 0 || MaskValue == -1);
283 if (A->getOpcode() == AndN2)
284 MaskValue = ~MaskValue;
285
286 if (!ReadsCond && A->registerDefIsDead(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
287 if (!MI.killsRegister(Reg: CondReg, TRI)) {
288 // Replace AND with MOV
289 if (MaskValue == 0) {
290 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
291 .addImm(Val: 0);
292 } else {
293 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
294 .addReg(RegNo: ExecReg);
295 }
296 }
297 // Remove AND instruction
298 A->eraseFromParent();
299 }
300
301 bool IsVCCZ = MI.getOpcode() == AMDGPU::S_CBRANCH_VCCZ;
302 if (SReg == ExecReg) {
303 // EXEC is updated directly
304 if (IsVCCZ) {
305 MI.eraseFromParent();
306 return true;
307 }
308 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
309 } else if (IsVCCZ && MaskValue == 0) {
310 // Will always branch
311 // Remove all successors shadowed by new unconditional branch
312 MachineBasicBlock *Parent = MI.getParent();
313 SmallVector<MachineInstr *, 4> ToRemove;
314 bool Found = false;
315 for (MachineInstr &Term : Parent->terminators()) {
316 if (Found) {
317 if (Term.isBranch())
318 ToRemove.push_back(Elt: &Term);
319 } else {
320 Found = Term.isIdenticalTo(Other: MI);
321 }
322 }
323 assert(Found && "conditional branch is not terminator");
324 for (auto *BranchMI : ToRemove) {
325 MachineOperand &Dst = BranchMI->getOperand(i: 0);
326 assert(Dst.isMBB() && "destination is not basic block");
327 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
328 Parent->removeSuccessor(Succ: Dst.getMBB());
329 BranchMI->eraseFromParent();
330 }
331
332 if (MachineBasicBlock *Succ = Parent->getFallThrough()) {
333 updateMLIBeforeRemovingEdge(From: Parent, To: Succ);
334 Parent->removeSuccessor(Succ);
335 }
336
337 // Rewrite to unconditional branch
338 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
339 } else if (!IsVCCZ && MaskValue == 0) {
340 // Will never branch
341 MachineOperand &Dst = MI.getOperand(i: 0);
342 assert(Dst.isMBB() && "destination is not basic block");
343 MachineBasicBlock *Parent = MI.getParent();
344 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
345 Parent->removeSuccessor(Succ: Dst.getMBB());
346 MI.eraseFromParent();
347 return true;
348 } else if (MaskValue == -1) {
349 // Depends only on EXEC
350 MI.setDesc(
351 TII->get(Opcode: IsVCCZ ? AMDGPU::S_CBRANCH_EXECZ : AMDGPU::S_CBRANCH_EXECNZ));
352 }
353
354 MI.removeOperand(OpNo: MI.findRegisterUseOperandIdx(Reg: CondReg, TRI, isKill: false /*Kill*/));
355 MI.addImplicitDefUseOperands(MF&: *MBB.getParent());
356
357 return true;
358}
359
360bool SIPreEmitPeephole::optimizeSetGPR(MachineInstr &First,
361 MachineInstr &MI) const {
362 MachineBasicBlock &MBB = *MI.getParent();
363 const MachineFunction &MF = *MBB.getParent();
364 const MachineRegisterInfo &MRI = MF.getRegInfo();
365 MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
366 Register IdxReg = Idx->isReg() ? Idx->getReg() : Register();
367 SmallVector<MachineInstr *, 4> ToRemove;
368 bool IdxOn = true;
369
370 if (!MI.isIdenticalTo(Other: First))
371 return false;
372
373 // Scan back to find an identical S_SET_GPR_IDX_ON
374 for (MachineBasicBlock::instr_iterator I = std::next(x: First.getIterator()),
375 E = MI.getIterator();
376 I != E; ++I) {
377 if (I->isBundle() || I->isDebugInstr())
378 continue;
379 switch (I->getOpcode()) {
380 case AMDGPU::S_SET_GPR_IDX_MODE:
381 return false;
382 case AMDGPU::S_SET_GPR_IDX_OFF:
383 IdxOn = false;
384 ToRemove.push_back(Elt: &*I);
385 break;
386 default:
387 if (I->modifiesRegister(Reg: AMDGPU::M0, TRI))
388 return false;
389 if (IdxReg && I->modifiesRegister(Reg: IdxReg, TRI))
390 return false;
391 if (llvm::any_of(Range: I->operands(), P: [&MRI, this](const MachineOperand &MO) {
392 return MO.isReg() && TRI->isVectorRegister(MRI, Reg: MO.getReg());
393 })) {
394 // The only exception allowed here is another indirect vector move
395 // with the same mode.
396 if (!IdxOn || !(I->getOpcode() == AMDGPU::V_MOV_B32_indirect_write ||
397 I->getOpcode() == AMDGPU::V_MOV_B32_indirect_read))
398 return false;
399 }
400 }
401 }
402
403 MI.eraseFromBundle();
404 for (MachineInstr *RI : ToRemove)
405 RI->eraseFromBundle();
406 return true;
407}
408
409bool SIPreEmitPeephole::getBlockDestinations(
410 MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB,
411 MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) {
412 if (TII->analyzeBranch(MBB&: SrcMBB, TBB&: TrueMBB, FBB&: FalseMBB, Cond))
413 return false;
414
415 if (!FalseMBB)
416 FalseMBB = SrcMBB.getNextNode();
417
418 return true;
419}
420
421namespace {
422class BranchWeightCostModel {
423 const SIInstrInfo &TII;
424 const TargetSchedModel &SchedModel;
425 BranchProbability BranchProb;
426 static constexpr uint64_t BranchNotTakenCost = 1;
427 uint64_t BranchTakenCost;
428 uint64_t ThenCyclesCost = 0;
429
430public:
431 BranchWeightCostModel(const SIInstrInfo &TII, const MachineInstr &Branch,
432 const MachineBasicBlock &Succ)
433 : TII(TII), SchedModel(TII.getSchedModel()) {
434 const MachineBasicBlock &Head = *Branch.getParent();
435 const auto *FromIt = find(Range: Head.successors(), Val: &Succ);
436 assert(FromIt != Head.succ_end());
437
438 BranchProb = Head.getSuccProbability(Succ: FromIt);
439 if (BranchProb.isUnknown())
440 BranchProb = BranchProbability::getZero();
441 BranchTakenCost = SchedModel.computeInstrLatency(MI: &Branch);
442 }
443
444 bool isProfitable(const MachineInstr &MI) {
445 if (TII.isWaitcnt(Opcode: MI.getOpcode()))
446 return false;
447
448 ThenCyclesCost += SchedModel.computeInstrLatency(MI: &MI);
449
450 // Consider `P = N/D` to be the probability of execz being false (skipping
451 // the then-block) The transformation is profitable if always executing the
452 // 'then' block is cheaper than executing sometimes 'then' and always
453 // executing s_cbranch_execz:
454 // * ThenCost <= P*ThenCost + (1-P)*BranchTakenCost + P*BranchNotTakenCost
455 // * (1-P) * ThenCost <= (1-P)*BranchTakenCost + P*BranchNotTakenCost
456 // * (D-N)/D * ThenCost <= (D-N)/D * BranchTakenCost + N/D *
457 // BranchNotTakenCost
458 uint64_t Numerator = BranchProb.getNumerator();
459 uint64_t Denominator = BranchProb.getDenominator();
460 return (Denominator - Numerator) * ThenCyclesCost <=
461 ((Denominator - Numerator) * BranchTakenCost +
462 Numerator * BranchNotTakenCost);
463 }
464};
465
466bool SIPreEmitPeephole::mustRetainExeczBranch(
467 const MachineInstr &Branch, const MachineBasicBlock &From,
468 const MachineBasicBlock &To) const {
469 assert(is_contained(Branch.getParent()->successors(), &From));
470 BranchWeightCostModel CostModel{*TII, Branch, From};
471
472 const MachineFunction *MF = From.getParent();
473 for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
474 MBBI != End && MBBI != ToI; ++MBBI) {
475 const MachineBasicBlock &MBB = *MBBI;
476
477 for (const MachineInstr &MI : MBB) {
478 // When a uniform loop is inside non-uniform control flow, the branch
479 // leaving the loop might never be taken when EXEC = 0.
480 // Hence we should retain cbranch out of the loop lest it become infinite.
481 if (MI.isConditionalBranch())
482 return true;
483
484 if (MI.isUnconditionalBranch() &&
485 TII->getBranchDestBlock(MI) != MBB.getNextNode())
486 return true;
487
488 if (MI.isMetaInstruction())
489 continue;
490
491 if (TII->hasUnwantedEffectsWhenEXECEmpty(MI))
492 return true;
493
494 if (!CostModel.isProfitable(MI))
495 return true;
496 }
497 }
498
499 return false;
500}
501} // namespace
502
503// Returns true if the skip branch instruction is removed.
504bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
505 MachineBasicBlock &SrcMBB) {
506
507 if (!TII->getSchedModel().hasInstrSchedModel())
508 return false;
509
510 MachineBasicBlock *TrueMBB = nullptr;
511 MachineBasicBlock *FalseMBB = nullptr;
512 SmallVector<MachineOperand, 1> Cond;
513
514 if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond))
515 return false;
516
517 // Consider only the forward branches.
518 if (SrcMBB.getNumber() >= TrueMBB->getNumber())
519 return false;
520
521 // Consider only when it is legal and profitable
522 if (mustRetainExeczBranch(Branch: MI, From: *FalseMBB, To: *TrueMBB))
523 return false;
524
525 LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
526 MI.eraseFromParent();
527 SrcMBB.removeSuccessor(Succ: TrueMBB);
528
529 return true;
530}
531
532/// Remove writes to the FP round mode and FP denorm mode that can never be
533/// observed: either the value written is already live in MODE, or a mode write
534/// replaces the whole mode field before anything reads it.
535///
536/// s_round_mode and s_denorm_mode each assign one field of MODE and preserve
537/// the rest of the register, so the two fields are tracked independently and a
538/// write to one is transparent to the other.
539///
540/// This is a purely intra-block analysis: the mode on entry to \p SrcMBB is
541/// unknown, and a write that is still live at the end of the block is kept for
542/// the benefit of the successors.
543bool SIPreEmitPeephole::removeRedundantModeWrites(
544 MachineBasicBlock &SrcMBB) const {
545 bool Changed = false;
546 ModeFieldState DenormMode;
547 ModeFieldState RoundMode;
548
549 for (MachineInstr &MI : make_early_inc_range(Range&: SrcMBB)) {
550 if (MI.isDebugInstr())
551 continue;
552
553 unsigned Opc = MI.getOpcode();
554 if (Opc == AMDGPU::S_DENORM_MODE || Opc == AMDGPU::S_ROUND_MODE) {
555 ModeFieldState &Field =
556 Opc == AMDGPU::S_DENORM_MODE ? DenormMode : RoundMode;
557 int64_t NewValue = MI.getOperand(i: 0).getImm();
558
559 if (Field.PendingWrite) {
560 LLVM_DEBUG(dbgs() << "Removing dead mode write: "
561 << *Field.PendingWrite);
562 Field.PendingWrite->eraseFromParent();
563 ++NumModeWritesRemoved;
564 Changed = true;
565 Field.PendingWrite = nullptr;
566 Field.Value = Field.ValueBeforePendingWrite;
567 }
568
569 if (Field.Value == NewValue) {
570 LLVM_DEBUG(dbgs() << "Removing redundant mode write: " << MI);
571 MI.eraseFromParent();
572 ++NumModeWritesRemoved;
573 Changed = true;
574 continue;
575 }
576
577 Field.ValueBeforePendingWrite = Field.Value;
578 Field.PendingWrite = &MI;
579 Field.Value = NewValue;
580 continue;
581 }
582
583 // Nothing tracked yet; skip register checks below.
584 if (!DenormMode.isTracked() && !RoundMode.isTracked())
585 continue;
586
587 // Inline asm cannot declare a MODE clobber, so assume it writes both.
588 if (MI.isInlineAsm() || MI.modifiesRegister(Reg: AMDGPU::MODE, TRI)) {
589 DenormMode = ModeFieldState();
590 RoundMode = ModeFieldState();
591 continue;
592 }
593
594 if (MI.readsRegister(Reg: AMDGPU::MODE, TRI) || MI.hasUnmodeledSideEffects()) {
595 DenormMode.PendingWrite = nullptr;
596 RoundMode.PendingWrite = nullptr;
597 }
598 }
599 return Changed;
600}
601
602bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
603 unsigned OpCode = MI.getOpcode();
604 Register DstReg = MI.getOperand(i: 0).getReg();
605 // Only the first register in the register pair needs to be checked due to the
606 // unpacking order. Packed instructions are unpacked such that the lower 32
607 // bits (i.e., the first register in the pair) are written first. This can
608 // introduce dependencies if the first register is written in one instruction
609 // and then read as part of the higher 32 bits in the subsequent instruction.
610 // Such scenarios can arise due to specific combinations of op_sel and
611 // op_sel_hi modifiers.
612 Register UnpackedDstReg = TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
613
614 const MachineOperand *Src0MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
615 if (Src0MO && Src0MO->isReg()) {
616 Register SrcReg0 = Src0MO->getReg();
617 unsigned Src0Mods =
618 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
619 Register HiSrc0Reg = (Src0Mods & SISrcMods::OP_SEL_1)
620 ? TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub1)
621 : TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub0);
622 // Check if the register selected by op_sel_hi is the same as the first
623 // register in the destination register pair.
624 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc0Reg))
625 return true;
626 }
627
628 const MachineOperand *Src1MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
629 if (Src1MO && Src1MO->isReg()) {
630 Register SrcReg1 = Src1MO->getReg();
631 unsigned Src1Mods =
632 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
633 Register HiSrc1Reg = (Src1Mods & SISrcMods::OP_SEL_1)
634 ? TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub1)
635 : TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub0);
636 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc1Reg))
637 return true;
638 }
639
640 // Applicable for packed instructions with 3 source operands, such as
641 // V_PK_FMA.
642 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
643 const MachineOperand *Src2MO =
644 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
645 if (Src2MO && Src2MO->isReg()) {
646 Register SrcReg2 = Src2MO->getReg();
647 unsigned Src2Mods =
648 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
649 Register HiSrc2Reg = (Src2Mods & SISrcMods::OP_SEL_1)
650 ? TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub1)
651 : TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub0);
652 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc2Reg))
653 return true;
654 }
655 }
656 return false;
657}
658
659uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
660 unsigned Opcode = I.getOpcode();
661 // Use 64 bit encoding to allow use of VOP3 instructions.
662 // VOP3 e64 instructions allow source modifiers
663 // e32 instructions don't allow source modifiers.
664 switch (Opcode) {
665 case AMDGPU::V_PK_ADD_F32:
666 case AMDGPU::V_PK_ADD_F32_gfx1250:
667 return AMDGPU::V_ADD_F32_e64;
668 case AMDGPU::V_PK_MUL_F32:
669 case AMDGPU::V_PK_MUL_F32_gfx1250:
670 return AMDGPU::V_MUL_F32_e64;
671 case AMDGPU::V_PK_FMA_F32:
672 case AMDGPU::V_PK_FMA_F32_gfx1250:
673 return AMDGPU::V_FMA_F32_e64;
674 default:
675 return std::numeric_limits<uint32_t>::max();
676 }
677 llvm_unreachable("Fully covered switch");
678}
679
680void SIPreEmitPeephole::addOperandAndMods(MachineInstrBuilder &NewMI,
681 unsigned SrcMods, bool IsHiBits,
682 const MachineOperand &SrcMO) {
683 unsigned NewSrcMods = 0;
684 unsigned NegModifier = IsHiBits ? SISrcMods::NEG_HI : SISrcMods::NEG;
685 unsigned OpSelModifier = IsHiBits ? SISrcMods::OP_SEL_1 : SISrcMods::OP_SEL_0;
686 // Packed instructions (VOP3P) do not support ABS. Hence, no checks are done
687 // for ABS modifiers.
688 // If NEG or NEG_HI is true, we need to negate the corresponding 32 bit
689 // lane.
690 // NEG_HI shares the same bit position with ABS. But packed instructions do
691 // not support ABS. Therefore, NEG_HI must be translated to NEG source
692 // modifier for the higher 32 bits. Unpacked VOP3 instructions support
693 // ABS, but do not support NEG_HI. Therefore we need to explicitly add the
694 // NEG modifier if present in the packed instruction.
695 if (SrcMods & NegModifier)
696 NewSrcMods |= SISrcMods::NEG;
697 // Src modifiers. Only negative modifiers are added if needed. Unpacked
698 // operations do not have op_sel, therefore it must be handled explicitly as
699 // done below.
700 NewMI.addImm(Val: NewSrcMods);
701 if (SrcMO.isImm()) {
702 NewMI.addImm(Val: SrcMO.getImm());
703 return;
704 }
705 // If op_sel == 0, select register 0 of reg:sub0_sub1.
706 Register UnpackedSrcReg = (SrcMods & OpSelModifier)
707 ? TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub1)
708 : TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub0);
709
710 MachineOperand UnpackedSrcMO =
711 MachineOperand::CreateReg(Reg: UnpackedSrcReg, /*isDef=*/false);
712 if (SrcMO.isKill()) {
713 // For each unpacked instruction, mark its source registers as killed if the
714 // corresponding source register in the original packed instruction was
715 // marked as killed.
716 //
717 // Exception:
718 // If the op_sel and op_sel_hi modifiers require both unpacked instructions
719 // to use the same register (e.g., due to overlapping access to low/high
720 // bits of the same packed register), then only the *second* (latter)
721 // instruction should mark the register as killed. This is because the
722 // second instruction handles the higher bits and is effectively the last
723 // user of the full register pair.
724
725 bool OpSel = SrcMods & SISrcMods::OP_SEL_0;
726 bool OpSelHi = SrcMods & SISrcMods::OP_SEL_1;
727 bool KillState = true;
728 if ((OpSel == OpSelHi) && !IsHiBits)
729 KillState = false;
730 UnpackedSrcMO.setIsKill(KillState);
731 }
732 NewMI.add(MO: UnpackedSrcMO);
733}
734
735void SIPreEmitPeephole::collectUnpackingCandidates(
736 MachineInstr &BeginMI, SetVector<MachineInstr *> &InstrsToUnpack,
737 uint16_t NumMFMACycles) {
738 auto *BB = BeginMI.getParent();
739 auto E = BB->end();
740 int TotalCyclesBetweenCandidates = 0;
741 auto SchedModel = TII->getSchedModel();
742 Register MFMADef = BeginMI.getOperand(i: 0).getReg();
743
744 for (auto I = std::next(x: BeginMI.getIterator()); I != E; ++I) {
745 MachineInstr &Instr = *I;
746 uint32_t UnpackedOpCode = mapToUnpackedOpcode(I&: Instr);
747 bool IsUnpackable =
748 !(UnpackedOpCode == std::numeric_limits<uint32_t>::max());
749 if (Instr.isMetaInstruction())
750 continue;
751 if ((Instr.isTerminator()) ||
752 (TII->isNeverCoissue(MI&: Instr) && !IsUnpackable) ||
753 (SIInstrInfo::modifiesModeRegister(MI: Instr) &&
754 Instr.modifiesRegister(Reg: AMDGPU::EXEC, TRI)))
755 return;
756
757 const MCSchedClassDesc *InstrSchedClassDesc =
758 SchedModel.resolveSchedClass(MI: &Instr);
759 uint16_t Latency =
760 SchedModel.getWriteProcResBegin(SC: InstrSchedClassDesc)->ReleaseAtCycle;
761 TotalCyclesBetweenCandidates += Latency;
762
763 if (TotalCyclesBetweenCandidates >= NumMFMACycles - 1)
764 return;
765 // Identify register dependencies between those used by the MFMA
766 // instruction and the following packed instructions. Also checks for
767 // transitive dependencies between the MFMA def and candidate instruction
768 // def and uses. Conservatively ensures that we do not incorrectly
769 // read/write registers.
770 for (const MachineOperand &InstrMO : Instr.operands()) {
771 if (!InstrMO.isReg() || !InstrMO.getReg().isValid())
772 continue;
773 if (TRI->regsOverlap(RegA: MFMADef, RegB: InstrMO.getReg()))
774 return;
775 }
776 if (!IsUnpackable)
777 continue;
778
779 if (canUnpackingClobberRegister(MI: Instr))
780 return;
781 // If it's a packed instruction, adjust latency: remove the packed
782 // latency, add latency of two unpacked instructions (currently estimated
783 // as 2 cycles).
784 TotalCyclesBetweenCandidates -= Latency;
785 // TODO: improve latency handling based on instruction modeling.
786 TotalCyclesBetweenCandidates += 2;
787 // Subtract 1 to account for MFMA issue latency.
788 if (TotalCyclesBetweenCandidates < NumMFMACycles - 1)
789 InstrsToUnpack.insert(X: &Instr);
790 }
791}
792
793void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
794 const MachineOperand &DstOp = I.getOperand(i: 0);
795
796 uint32_t UnpackedOpcode = mapToUnpackedOpcode(I);
797 assert(UnpackedOpcode != std::numeric_limits<uint32_t>::max() &&
798 "Unsupported Opcode");
799
800 MachineInstrBuilder Op0LOp1L =
801 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
802 MachineOperand LoDstOp = Op0LOp1L->getOperand(i: 0);
803
804 LoDstOp.setIsUndef(DstOp.isUndef());
805
806 MachineInstrBuilder Op0HOp1H =
807 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
808 MachineOperand HiDstOp = Op0HOp1H->getOperand(i: 0);
809
810 uint32_t IFlags = I.getFlags();
811 Op0LOp1L->setFlags(IFlags);
812 Op0HOp1H->setFlags(IFlags);
813 LoDstOp.setIsRenamable(DstOp.isRenamable());
814 HiDstOp.setIsRenamable(DstOp.isRenamable());
815
816 I.eraseFromParent();
817}
818
819MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
820 uint32_t UnpackedOpcode,
821 bool IsHiBits) {
822 MachineBasicBlock &MBB = *I.getParent();
823 const DebugLoc &DL = I.getDebugLoc();
824 const MachineOperand *SrcMO0 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0);
825 const MachineOperand *SrcMO1 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1);
826 Register DstReg = I.getOperand(i: 0).getReg();
827 unsigned OpCode = I.getOpcode();
828 Register UnpackedDstReg = IsHiBits ? TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub1)
829 : TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
830
831 int64_t ClampVal = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::clamp)->getImm();
832 unsigned Src0Mods =
833 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
834 unsigned Src1Mods =
835 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
836
837 MachineInstrBuilder NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: UnpackedOpcode));
838 NewMI.addDef(RegNo: UnpackedDstReg); // vdst
839 addOperandAndMods(NewMI, SrcMods: Src0Mods, IsHiBits, SrcMO: *SrcMO0);
840 addOperandAndMods(NewMI, SrcMods: Src1Mods, IsHiBits, SrcMO: *SrcMO1);
841
842 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
843 const MachineOperand *SrcMO2 =
844 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2);
845 unsigned Src2Mods =
846 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
847 addOperandAndMods(NewMI, SrcMods: Src2Mods, IsHiBits, SrcMO: *SrcMO2);
848 }
849 NewMI.addImm(Val: ClampVal); // clamp
850 // Packed instructions do not support output modifiers. safe to assign them 0
851 // for this use case
852 NewMI.addImm(Val: 0); // omod
853 return NewMI;
854}
855
856PreservedAnalyses
857llvm::SIPreEmitPeepholePass::run(MachineFunction &MF,
858 MachineFunctionAnalysisManager &MFAM) {
859 auto *MLI = MFAM.getCachedResult<MachineLoopAnalysis>(IR&: MF);
860 SIPreEmitPeephole Impl;
861
862 if (Impl.run(MF, MLI)) {
863 auto PA = getMachineFunctionPassPreservedAnalyses();
864 PA.preserve<MachineLoopAnalysis>();
865 return PA;
866 }
867
868 return PreservedAnalyses::all();
869}
870
871bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
872 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
873 TII = ST.getInstrInfo();
874 TRI = &TII->getRegisterInfo();
875 MLI = LoopInfo;
876 bool Changed = false;
877
878 MF.RenumberBlocks();
879
880 for (MachineBasicBlock &MBB : MF) {
881 Changed |= removeRedundantModeWrites(SrcMBB&: MBB);
882
883 MachineBasicBlock::iterator TermI = MBB.getFirstTerminator();
884 // Check first terminator for branches to optimize
885 if (TermI != MBB.end()) {
886 MachineInstr &MI = *TermI;
887 switch (MI.getOpcode()) {
888 case AMDGPU::S_CBRANCH_VCCZ:
889 case AMDGPU::S_CBRANCH_VCCNZ:
890 Changed |= optimizeVccBranch(MI);
891 break;
892 case AMDGPU::S_CBRANCH_EXECZ:
893 Changed |= removeExeczBranch(MI, SrcMBB&: MBB);
894 break;
895 }
896 }
897
898 if (!ST.hasVGPRIndexMode())
899 continue;
900
901 MachineInstr *SetGPRMI = nullptr;
902 const unsigned Threshold = 20;
903 unsigned Count = 0;
904 // Scan the block for two S_SET_GPR_IDX_ON instructions to see if a
905 // second is not needed. Do expensive checks in the optimizeSetGPR()
906 // and limit the distance to 20 instructions for compile time purposes.
907 // Note: this needs to work on bundles as S_SET_GPR_IDX* instructions
908 // may be bundled with the instructions they modify.
909 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
910 if (Count == Threshold)
911 SetGPRMI = nullptr;
912 else
913 ++Count;
914
915 if (MI.getOpcode() != AMDGPU::S_SET_GPR_IDX_ON)
916 continue;
917
918 Count = 0;
919 if (!SetGPRMI) {
920 SetGPRMI = &MI;
921 continue;
922 }
923
924 if (optimizeSetGPR(First&: *SetGPRMI, MI))
925 Changed = true;
926 else
927 SetGPRMI = &MI;
928 }
929 }
930
931 // TODO: Fold this into previous block, if possible. Evaluate and handle any
932 // side effects.
933
934 // Perform the extra MF scans only for supported archs
935 if (!ST.hasGFX940Insts())
936 return Changed;
937 for (MachineBasicBlock &MBB : MF) {
938 // Unpack packed instructions overlapped by MFMAs. This allows the
939 // compiler to co-issue unpacked instructions with MFMA
940 auto SchedModel = TII->getSchedModel();
941 SetVector<MachineInstr *> InstrsToUnpack;
942 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
943 if (!SIInstrInfo::isMFMA(MI))
944 continue;
945 const MCSchedClassDesc *SchedClassDesc =
946 SchedModel.resolveSchedClass(MI: &MI);
947 uint16_t NumMFMACycles =
948 SchedModel.getWriteProcResBegin(SC: SchedClassDesc)->ReleaseAtCycle;
949 collectUnpackingCandidates(BeginMI&: MI, InstrsToUnpack, NumMFMACycles);
950 }
951 for (MachineInstr *MI : InstrsToUnpack) {
952 performF32Unpacking(I&: *MI);
953 }
954 }
955
956 return Changed;
957}
958