1//===-- SIPreEmitPeephole.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass performs the peephole optimizations before code emission.
11///
12/// Additionally, this pass also unpacks packed instructions (V_PK_MUL_F32/F16,
13/// V_PK_ADD_F32/F16, V_PK_FMA_F32) adjacent to MFMAs such that they can be
14/// co-issued. This helps with overlapping MFMA and certain vector instructions
15/// in machine schedules and is expected to improve performance. Only those
16/// packed instructions are unpacked that are overlapped by the MFMA latency.
17/// Rest should remain untouched.
18/// TODO: Add support for F16 packed instructions
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "GCNSubtarget.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/CodeGen/MachineDominators.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachineLoopInfo.h"
27#include "llvm/CodeGen/TargetSchedule.h"
28#include "llvm/Support/BranchProbability.h"
29using namespace llvm;
30
31#define DEBUG_TYPE "si-pre-emit-peephole"
32
33STATISTIC(NumModeWritesRemoved,
34 "Number of redundant mode register writes removed");
35
36namespace {
37
38/// The state of one independent field of the MODE register, as tracked by
39/// removeRedundantModeWrites.
40struct ModeFieldState {
41 std::optional<int64_t> Value;
42 std::optional<int64_t> ValueBeforePendingWrite;
43 MachineInstr *PendingWrite = nullptr;
44
45 bool isTracked() const { return PendingWrite || Value; }
46};
47
48class SIPreEmitPeephole {
49private:
50 const SIInstrInfo *TII = nullptr;
51 const SIRegisterInfo *TRI = nullptr;
52 MachineLoopInfo *MLI = nullptr;
53
54 bool optimizeVccBranch(MachineInstr &MI) const;
55 void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
56 MachineBasicBlock *To) const;
57 bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
58 bool getBlockDestinations(MachineBasicBlock &SrcMBB,
59 MachineBasicBlock *&TrueMBB,
60 MachineBasicBlock *&FalseMBB,
61 SmallVectorImpl<MachineOperand> &Cond);
62 bool mustRetainExeczBranch(const MachineInstr &Branch,
63 const MachineBasicBlock &From,
64 const MachineBasicBlock &To) const;
65 bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
66 bool removeRedundantModeWrites(MachineBasicBlock &SrcMBB) const;
67 // Creates a list of packed instructions following an MFMA that are suitable
68 // for unpacking.
69 void collectUnpackingCandidates(MachineInstr &BeginMI,
70 SetVector<MachineInstr *> &InstrsToUnpack,
71 uint16_t NumMFMACycles);
72 // v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[2:3] op_sel:[1,1,1]
73 // op_sel_hi:[0,0,0]
74 // ==>
75 // v_fma_f32 v0, v1, v3, v3
76 // v_fma_f32 v1, v0, v2, v2
77 // Here, we have overwritten v0 before we use it. This function checks if
78 // unpacking can lead to such a situation.
79 bool canUnpackingClobberRegister(const MachineInstr &MI);
80 // Unpack and insert F32 packed instructions, such as V_PK_MUL, V_PK_ADD, and
81 // V_PK_FMA. Currently, only V_PK_MUL, V_PK_ADD, V_PK_FMA are supported for
82 // this transformation.
83 void performF32Unpacking(MachineInstr &I);
84 // Select corresponding unpacked instruction
85 uint32_t mapToUnpackedOpcode(MachineInstr &I);
86 // Creates the unpacked instruction to be inserted. Adds source modifiers to
87 // the unpacked instructions based on the source modifiers in the packed
88 // instruction.
89 MachineInstrBuilder createUnpackedMI(MachineInstr &I, uint32_t UnpackedOpcode,
90 bool IsHiBits);
91 // Process operands/source modifiers from packed instructions and insert the
92 // appropriate source modifers and operands into the unpacked instructions.
93 void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
94 bool IsHiBits, const MachineOperand &SrcMO);
95
96public:
97 bool run(MachineFunction &MF, MachineLoopInfo *MLI);
98};
99
100class SIPreEmitPeepholeLegacy : public MachineFunctionPass {
101public:
102 static char ID;
103
104 SIPreEmitPeepholeLegacy() : MachineFunctionPass(ID) {}
105
106 void getAnalysisUsage(AnalysisUsage &AU) const override {
107 AU.addUsedIfAvailable<MachineLoopInfoWrapperPass>();
108 AU.addPreserved<MachineLoopInfoWrapperPass>();
109 MachineFunctionPass::getAnalysisUsage(AU);
110 }
111
112 bool runOnMachineFunction(MachineFunction &MF) override {
113 auto *MLIWrapper = getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
114 MachineLoopInfo *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
115 return SIPreEmitPeephole().run(MF, MLI);
116 }
117};
118
119} // End anonymous namespace.
120
121INITIALIZE_PASS(SIPreEmitPeepholeLegacy, DEBUG_TYPE,
122 "SI peephole optimizations", false, false)
123
124char SIPreEmitPeepholeLegacy::ID = 0;
125
126char &llvm::SIPreEmitPeepholeID = SIPreEmitPeepholeLegacy::ID;
127
128void SIPreEmitPeephole::updateMLIBeforeRemovingEdge(
129 MachineBasicBlock *From, MachineBasicBlock *To) const {
130 if (!MLI)
131 return;
132
133 // Only handle back-edges: To must be a loop header with From inside the loop.
134 MachineLoop *Loop = MLI->getLoopFor(BB: To);
135 if (!Loop || Loop->getHeader() != To || !Loop->contains(BB: From))
136 return;
137
138 // Count back-edges
139 unsigned BackEdgeCount = 0;
140 for (MachineBasicBlock *Pred : To->predecessors()) {
141 if (Loop->contains(BB: Pred))
142 BackEdgeCount++;
143 }
144
145 if (BackEdgeCount > 1)
146 return;
147
148 MachineLoop *ParentLoop = Loop->getParentLoop();
149
150 // Re-map blocks directly owned by this loop to the parent.
151 for (MachineBasicBlock *BB : Loop->blocks()) {
152 if (MLI->getLoopFor(BB) == Loop)
153 MLI->changeLoopFor(BB, L: ParentLoop);
154 }
155
156 // Reparent all child loops.
157 while (!Loop->isInnermost()) {
158 MachineLoop *Child = Loop->removeChildLoop(I: std::prev(x: Loop->end()));
159 if (ParentLoop)
160 ParentLoop->addChildLoop(NewChild: Child);
161 else
162 MLI->addTopLevelLoop(New: Child);
163 }
164
165 if (ParentLoop)
166 ParentLoop->removeChildLoop(Child: Loop);
167 else
168 MLI->removeLoop(I: llvm::find(Range&: *MLI, Val: Loop));
169
170 MLI->destroy(L: Loop);
171}
172
173bool SIPreEmitPeephole::optimizeVccBranch(MachineInstr &MI) const {
174 // Match:
175 // sreg = -1 or 0
176 // vcc = S_AND_B64 exec, sreg or S_ANDN2_B64 exec, sreg
177 // S_CBRANCH_VCC[N]Z
178 // =>
179 // S_CBRANCH_EXEC[N]Z
180 // We end up with this pattern sometimes after basic block placement.
181 // It happens while combining a block which assigns -1 or 0 to a saved mask
182 // and another block which consumes that saved mask and then a branch.
183 //
184 // While searching this also performs the following substitution:
185 // vcc = V_CMP
186 // vcc = S_AND exec, vcc
187 // S_CBRANCH_VCC[N]Z
188 // =>
189 // vcc = V_CMP
190 // S_CBRANCH_VCC[N]Z
191
192 bool Changed = false;
193 MachineBasicBlock &MBB = *MI.getParent();
194 const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
195 const bool IsWave32 = ST.isWave32();
196 const unsigned CondReg = TRI->getVCC();
197 const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
198 const unsigned And = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
199 const unsigned AndN2 = IsWave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64;
200 const unsigned Mov = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
201
202 MachineBasicBlock::reverse_iterator A = MI.getReverseIterator(),
203 E = MBB.rend();
204 bool ReadsCond = false;
205 unsigned Threshold = 5;
206 for (++A; A != E; ++A) {
207 if (!--Threshold)
208 return false;
209 if (A->modifiesRegister(Reg: ExecReg, TRI))
210 return false;
211 if (A->modifiesRegister(Reg: CondReg, TRI)) {
212 if (!A->definesRegister(Reg: CondReg, TRI) ||
213 (A->getOpcode() != And && A->getOpcode() != AndN2))
214 return false;
215 break;
216 }
217 ReadsCond |= A->readsRegister(Reg: CondReg, TRI);
218 }
219 if (A == E)
220 return false;
221
222 MachineOperand &Op1 = A->getOperand(i: 1);
223 MachineOperand &Op2 = A->getOperand(i: 2);
224 if ((!Op1.isReg() || Op1.getReg() != ExecReg) && Op2.isReg() &&
225 Op2.getReg() == ExecReg) {
226 TII->commuteInstruction(MI&: *A);
227 Changed = true;
228 }
229 if (!Op1.isReg() || Op1.getReg() != ExecReg)
230 return Changed;
231 if (Op2.isImm() && !(Op2.getImm() == -1 || Op2.getImm() == 0))
232 return Changed;
233
234 int64_t MaskValue = 0;
235 Register SReg;
236 if (Op2.isReg()) {
237 SReg = Op2.getReg();
238 auto M = std::next(x: A);
239 bool ReadsSreg = false;
240 bool ModifiesExec = false;
241 for (; M != E; ++M) {
242 if (M->definesRegister(Reg: SReg, TRI))
243 break;
244 if (M->modifiesRegister(Reg: SReg, TRI))
245 return Changed;
246 ReadsSreg |= M->readsRegister(Reg: SReg, TRI);
247 ModifiesExec |= M->modifiesRegister(Reg: ExecReg, TRI);
248 }
249 if (M == E)
250 return Changed;
251 // If SReg is VCC and SReg definition is a VALU comparison.
252 // This means S_AND with EXEC is not required, unless
253 // the implicit def of SCC is alive.
254 // Erase the S_AND and return.
255 // Note: isVOPC is used instead of isCompare to catch V_CMP_CLASS
256 if (A->getOpcode() == And && SReg == CondReg && !ModifiesExec &&
257 TII->isVOPC(MI: *M) && A->allImplicitDefsAreDead()) {
258 A->eraseFromParent();
259 return true;
260 }
261
262 if (!M->isMoveImmediate() || !M->getOperand(i: 1).isImm() ||
263 (M->getOperand(i: 1).getImm() != -1 && M->getOperand(i: 1).getImm() != 0))
264 return Changed;
265 MaskValue = M->getOperand(i: 1).getImm();
266 // First if sreg is only used in the AND instruction fold the immediate
267 // into the AND.
268 if (!ReadsSreg && Op2.isKill()) {
269 A->getOperand(i: 2).ChangeToImmediate(ImmVal: MaskValue);
270 M->eraseFromParent();
271 }
272 } else if (Op2.isImm()) {
273 MaskValue = Op2.getImm();
274 } else {
275 llvm_unreachable("Op2 must be register or immediate");
276 }
277
278 // Invert mask for s_andn2
279 assert(MaskValue == 0 || MaskValue == -1);
280 if (A->getOpcode() == AndN2)
281 MaskValue = ~MaskValue;
282
283 if (!ReadsCond && A->registerDefIsDead(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
284 if (!MI.killsRegister(Reg: CondReg, TRI)) {
285 // Replace AND with MOV
286 if (MaskValue == 0) {
287 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
288 .addImm(Val: 0);
289 } else {
290 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
291 .addReg(RegNo: ExecReg);
292 }
293 }
294 // Remove AND instruction
295 A->eraseFromParent();
296 }
297
298 bool IsVCCZ = MI.getOpcode() == AMDGPU::S_CBRANCH_VCCZ;
299 if (SReg == ExecReg) {
300 // EXEC is updated directly
301 if (IsVCCZ) {
302 MI.eraseFromParent();
303 return true;
304 }
305 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
306 } else if (IsVCCZ && MaskValue == 0) {
307 // Will always branch
308 // Remove all successors shadowed by new unconditional branch
309 MachineBasicBlock *Parent = MI.getParent();
310 SmallVector<MachineInstr *, 4> ToRemove;
311 bool Found = false;
312 for (MachineInstr &Term : Parent->terminators()) {
313 if (Found) {
314 if (Term.isBranch())
315 ToRemove.push_back(Elt: &Term);
316 } else {
317 Found = Term.isIdenticalTo(Other: MI);
318 }
319 }
320 assert(Found && "conditional branch is not terminator");
321 for (auto *BranchMI : ToRemove) {
322 MachineOperand &Dst = BranchMI->getOperand(i: 0);
323 assert(Dst.isMBB() && "destination is not basic block");
324 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
325 Parent->removeSuccessor(Succ: Dst.getMBB());
326 BranchMI->eraseFromParent();
327 }
328
329 if (MachineBasicBlock *Succ = Parent->getFallThrough()) {
330 updateMLIBeforeRemovingEdge(From: Parent, To: Succ);
331 Parent->removeSuccessor(Succ);
332 }
333
334 // Rewrite to unconditional branch
335 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
336 } else if (!IsVCCZ && MaskValue == 0) {
337 // Will never branch
338 MachineOperand &Dst = MI.getOperand(i: 0);
339 assert(Dst.isMBB() && "destination is not basic block");
340 MachineBasicBlock *Parent = MI.getParent();
341 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
342 Parent->removeSuccessor(Succ: Dst.getMBB());
343 MI.eraseFromParent();
344 return true;
345 } else if (MaskValue == -1) {
346 // Depends only on EXEC
347 MI.setDesc(
348 TII->get(Opcode: IsVCCZ ? AMDGPU::S_CBRANCH_EXECZ : AMDGPU::S_CBRANCH_EXECNZ));
349 }
350
351 MI.removeOperand(OpNo: MI.findRegisterUseOperandIdx(Reg: CondReg, TRI, isKill: false /*Kill*/));
352 MI.addImplicitDefUseOperands(MF&: *MBB.getParent());
353
354 return true;
355}
356
357bool SIPreEmitPeephole::optimizeSetGPR(MachineInstr &First,
358 MachineInstr &MI) const {
359 MachineBasicBlock &MBB = *MI.getParent();
360 const MachineFunction &MF = *MBB.getParent();
361 const MachineRegisterInfo &MRI = MF.getRegInfo();
362 MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
363 Register IdxReg = Idx->isReg() ? Idx->getReg() : Register();
364 SmallVector<MachineInstr *, 4> ToRemove;
365 bool IdxOn = true;
366
367 if (!MI.isIdenticalTo(Other: First))
368 return false;
369
370 // Scan back to find an identical S_SET_GPR_IDX_ON
371 for (MachineBasicBlock::instr_iterator I = std::next(x: First.getIterator()),
372 E = MI.getIterator();
373 I != E; ++I) {
374 if (I->isBundle() || I->isDebugInstr())
375 continue;
376 switch (I->getOpcode()) {
377 case AMDGPU::S_SET_GPR_IDX_MODE:
378 return false;
379 case AMDGPU::S_SET_GPR_IDX_OFF:
380 IdxOn = false;
381 ToRemove.push_back(Elt: &*I);
382 break;
383 default:
384 if (I->modifiesRegister(Reg: AMDGPU::M0, TRI))
385 return false;
386 if (IdxReg && I->modifiesRegister(Reg: IdxReg, TRI))
387 return false;
388 if (llvm::any_of(Range: I->operands(), P: [&MRI, this](const MachineOperand &MO) {
389 return MO.isReg() && TRI->isVectorRegister(MRI, Reg: MO.getReg());
390 })) {
391 // The only exception allowed here is another indirect vector move
392 // with the same mode.
393 if (!IdxOn || !(I->getOpcode() == AMDGPU::V_MOV_B32_indirect_write ||
394 I->getOpcode() == AMDGPU::V_MOV_B32_indirect_read))
395 return false;
396 }
397 }
398 }
399
400 MI.eraseFromBundle();
401 for (MachineInstr *RI : ToRemove)
402 RI->eraseFromBundle();
403 return true;
404}
405
406bool SIPreEmitPeephole::getBlockDestinations(
407 MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB,
408 MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) {
409 if (TII->analyzeBranch(MBB&: SrcMBB, TBB&: TrueMBB, FBB&: FalseMBB, Cond))
410 return false;
411
412 if (!FalseMBB)
413 FalseMBB = SrcMBB.getNextNode();
414
415 return true;
416}
417
418namespace {
419class BranchWeightCostModel {
420 const SIInstrInfo &TII;
421 const TargetSchedModel &SchedModel;
422 BranchProbability BranchProb;
423 static constexpr uint64_t BranchNotTakenCost = 1;
424 uint64_t BranchTakenCost;
425 uint64_t ThenCyclesCost = 0;
426
427public:
428 BranchWeightCostModel(const SIInstrInfo &TII, const MachineInstr &Branch,
429 const MachineBasicBlock &Succ)
430 : TII(TII), SchedModel(TII.getSchedModel()) {
431 const MachineBasicBlock &Head = *Branch.getParent();
432 const auto *FromIt = find(Range: Head.successors(), Val: &Succ);
433 assert(FromIt != Head.succ_end());
434
435 BranchProb = Head.getSuccProbability(Succ: FromIt);
436 if (BranchProb.isUnknown())
437 BranchProb = BranchProbability::getZero();
438 BranchTakenCost = SchedModel.computeInstrLatency(MI: &Branch);
439 }
440
441 bool isProfitable(const MachineInstr &MI) {
442 if (TII.isWaitcnt(Opcode: MI.getOpcode()))
443 return false;
444
445 ThenCyclesCost += SchedModel.computeInstrLatency(MI: &MI);
446
447 // Consider `P = N/D` to be the probability of execz being false (skipping
448 // the then-block) The transformation is profitable if always executing the
449 // 'then' block is cheaper than executing sometimes 'then' and always
450 // executing s_cbranch_execz:
451 // * ThenCost <= P*ThenCost + (1-P)*BranchTakenCost + P*BranchNotTakenCost
452 // * (1-P) * ThenCost <= (1-P)*BranchTakenCost + P*BranchNotTakenCost
453 // * (D-N)/D * ThenCost <= (D-N)/D * BranchTakenCost + N/D *
454 // BranchNotTakenCost
455 uint64_t Numerator = BranchProb.getNumerator();
456 uint64_t Denominator = BranchProb.getDenominator();
457 return (Denominator - Numerator) * ThenCyclesCost <=
458 ((Denominator - Numerator) * BranchTakenCost +
459 Numerator * BranchNotTakenCost);
460 }
461};
462
463bool SIPreEmitPeephole::mustRetainExeczBranch(
464 const MachineInstr &Branch, const MachineBasicBlock &From,
465 const MachineBasicBlock &To) const {
466 assert(is_contained(Branch.getParent()->successors(), &From));
467 BranchWeightCostModel CostModel{*TII, Branch, From};
468
469 const MachineFunction *MF = From.getParent();
470 for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
471 MBBI != End && MBBI != ToI; ++MBBI) {
472 const MachineBasicBlock &MBB = *MBBI;
473
474 for (const MachineInstr &MI : MBB) {
475 // When a uniform loop is inside non-uniform control flow, the branch
476 // leaving the loop might never be taken when EXEC = 0.
477 // Hence we should retain cbranch out of the loop lest it become infinite.
478 if (MI.isConditionalBranch())
479 return true;
480
481 if (MI.isUnconditionalBranch() &&
482 TII->getBranchDestBlock(MI) != MBB.getNextNode())
483 return true;
484
485 if (MI.isMetaInstruction())
486 continue;
487
488 if (TII->hasUnwantedEffectsWhenEXECEmpty(MI))
489 return true;
490
491 if (!CostModel.isProfitable(MI))
492 return true;
493 }
494 }
495
496 return false;
497}
498} // namespace
499
500// Returns true if the skip branch instruction is removed.
501bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
502 MachineBasicBlock &SrcMBB) {
503
504 if (!TII->getSchedModel().hasInstrSchedModel())
505 return false;
506
507 MachineBasicBlock *TrueMBB = nullptr;
508 MachineBasicBlock *FalseMBB = nullptr;
509 SmallVector<MachineOperand, 1> Cond;
510
511 if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond))
512 return false;
513
514 // Consider only the forward branches.
515 if (SrcMBB.getNumber() >= TrueMBB->getNumber())
516 return false;
517
518 // Consider only when it is legal and profitable
519 if (mustRetainExeczBranch(Branch: MI, From: *FalseMBB, To: *TrueMBB))
520 return false;
521
522 LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
523 MI.eraseFromParent();
524 SrcMBB.removeSuccessor(Succ: TrueMBB);
525
526 return true;
527}
528
529/// Remove writes to the FP round mode and FP denorm mode that can never be
530/// observed: either the value written is already live in MODE, or a mode write
531/// replaces the whole mode field before anything reads it.
532///
533/// s_round_mode and s_denorm_mode each assign one field of MODE and preserve
534/// the rest of the register, so the two fields are tracked independently and a
535/// write to one is transparent to the other.
536///
537/// This is a purely intra-block analysis: the mode on entry to \p SrcMBB is
538/// unknown, and a write that is still live at the end of the block is kept for
539/// the benefit of the successors.
540bool SIPreEmitPeephole::removeRedundantModeWrites(
541 MachineBasicBlock &SrcMBB) const {
542 bool Changed = false;
543 ModeFieldState DenormMode;
544 ModeFieldState RoundMode;
545
546 for (MachineInstr &MI : make_early_inc_range(Range&: SrcMBB)) {
547 if (MI.isDebugInstr())
548 continue;
549
550 unsigned Opc = MI.getOpcode();
551 if (Opc == AMDGPU::S_DENORM_MODE || Opc == AMDGPU::S_ROUND_MODE) {
552 ModeFieldState &Field =
553 Opc == AMDGPU::S_DENORM_MODE ? DenormMode : RoundMode;
554 int64_t NewValue = MI.getOperand(i: 0).getImm();
555
556 if (Field.PendingWrite) {
557 LLVM_DEBUG(dbgs() << "Removing dead mode write: "
558 << *Field.PendingWrite);
559 Field.PendingWrite->eraseFromParent();
560 ++NumModeWritesRemoved;
561 Changed = true;
562 Field.PendingWrite = nullptr;
563 Field.Value = Field.ValueBeforePendingWrite;
564 }
565
566 if (Field.Value == NewValue) {
567 LLVM_DEBUG(dbgs() << "Removing redundant mode write: " << MI);
568 MI.eraseFromParent();
569 ++NumModeWritesRemoved;
570 Changed = true;
571 continue;
572 }
573
574 Field.ValueBeforePendingWrite = Field.Value;
575 Field.PendingWrite = &MI;
576 Field.Value = NewValue;
577 continue;
578 }
579
580 // Nothing tracked yet; skip register checks below.
581 if (!DenormMode.isTracked() && !RoundMode.isTracked())
582 continue;
583
584 // Inline asm cannot declare a MODE clobber, so assume it writes both.
585 if (MI.isInlineAsm() || MI.modifiesRegister(Reg: AMDGPU::MODE, TRI)) {
586 DenormMode = ModeFieldState();
587 RoundMode = ModeFieldState();
588 continue;
589 }
590
591 if (MI.readsRegister(Reg: AMDGPU::MODE, TRI) || MI.hasUnmodeledSideEffects()) {
592 DenormMode.PendingWrite = nullptr;
593 RoundMode.PendingWrite = nullptr;
594 }
595 }
596 return Changed;
597}
598
599bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
600 unsigned OpCode = MI.getOpcode();
601 Register DstReg = MI.getOperand(i: 0).getReg();
602 // Only the first register in the register pair needs to be checked due to the
603 // unpacking order. Packed instructions are unpacked such that the lower 32
604 // bits (i.e., the first register in the pair) are written first. This can
605 // introduce dependencies if the first register is written in one instruction
606 // and then read as part of the higher 32 bits in the subsequent instruction.
607 // Such scenarios can arise due to specific combinations of op_sel and
608 // op_sel_hi modifiers.
609 Register UnpackedDstReg = TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
610
611 const MachineOperand *Src0MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
612 if (Src0MO && Src0MO->isReg()) {
613 Register SrcReg0 = Src0MO->getReg();
614 unsigned Src0Mods =
615 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
616 Register HiSrc0Reg = (Src0Mods & SISrcMods::OP_SEL_1)
617 ? TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub1)
618 : TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub0);
619 // Check if the register selected by op_sel_hi is the same as the first
620 // register in the destination register pair.
621 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc0Reg))
622 return true;
623 }
624
625 const MachineOperand *Src1MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
626 if (Src1MO && Src1MO->isReg()) {
627 Register SrcReg1 = Src1MO->getReg();
628 unsigned Src1Mods =
629 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
630 Register HiSrc1Reg = (Src1Mods & SISrcMods::OP_SEL_1)
631 ? TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub1)
632 : TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub0);
633 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc1Reg))
634 return true;
635 }
636
637 // Applicable for packed instructions with 3 source operands, such as
638 // V_PK_FMA.
639 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
640 const MachineOperand *Src2MO =
641 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
642 if (Src2MO && Src2MO->isReg()) {
643 Register SrcReg2 = Src2MO->getReg();
644 unsigned Src2Mods =
645 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
646 Register HiSrc2Reg = (Src2Mods & SISrcMods::OP_SEL_1)
647 ? TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub1)
648 : TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub0);
649 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc2Reg))
650 return true;
651 }
652 }
653 return false;
654}
655
656uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
657 unsigned Opcode = I.getOpcode();
658 // Use 64 bit encoding to allow use of VOP3 instructions.
659 // VOP3 e64 instructions allow source modifiers
660 // e32 instructions don't allow source modifiers.
661 switch (Opcode) {
662 case AMDGPU::V_PK_ADD_F32:
663 case AMDGPU::V_PK_ADD_F32_gfx1250:
664 return AMDGPU::V_ADD_F32_e64;
665 case AMDGPU::V_PK_MUL_F32:
666 case AMDGPU::V_PK_MUL_F32_gfx1250:
667 return AMDGPU::V_MUL_F32_e64;
668 case AMDGPU::V_PK_FMA_F32:
669 case AMDGPU::V_PK_FMA_F32_gfx1250:
670 return AMDGPU::V_FMA_F32_e64;
671 default:
672 return std::numeric_limits<uint32_t>::max();
673 }
674 llvm_unreachable("Fully covered switch");
675}
676
677void SIPreEmitPeephole::addOperandAndMods(MachineInstrBuilder &NewMI,
678 unsigned SrcMods, bool IsHiBits,
679 const MachineOperand &SrcMO) {
680 unsigned NewSrcMods = 0;
681 unsigned NegModifier = IsHiBits ? SISrcMods::NEG_HI : SISrcMods::NEG;
682 unsigned OpSelModifier = IsHiBits ? SISrcMods::OP_SEL_1 : SISrcMods::OP_SEL_0;
683 // Packed instructions (VOP3P) do not support ABS. Hence, no checks are done
684 // for ABS modifiers.
685 // If NEG or NEG_HI is true, we need to negate the corresponding 32 bit
686 // lane.
687 // NEG_HI shares the same bit position with ABS. But packed instructions do
688 // not support ABS. Therefore, NEG_HI must be translated to NEG source
689 // modifier for the higher 32 bits. Unpacked VOP3 instructions support
690 // ABS, but do not support NEG_HI. Therefore we need to explicitly add the
691 // NEG modifier if present in the packed instruction.
692 if (SrcMods & NegModifier)
693 NewSrcMods |= SISrcMods::NEG;
694 // Src modifiers. Only negative modifiers are added if needed. Unpacked
695 // operations do not have op_sel, therefore it must be handled explicitly as
696 // done below.
697 NewMI.addImm(Val: NewSrcMods);
698 if (SrcMO.isImm()) {
699 NewMI.addImm(Val: SrcMO.getImm());
700 return;
701 }
702 // If op_sel == 0, select register 0 of reg:sub0_sub1.
703 Register UnpackedSrcReg = (SrcMods & OpSelModifier)
704 ? TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub1)
705 : TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub0);
706
707 MachineOperand UnpackedSrcMO =
708 MachineOperand::CreateReg(Reg: UnpackedSrcReg, /*isDef=*/false);
709 if (SrcMO.isKill()) {
710 // For each unpacked instruction, mark its source registers as killed if the
711 // corresponding source register in the original packed instruction was
712 // marked as killed.
713 //
714 // Exception:
715 // If the op_sel and op_sel_hi modifiers require both unpacked instructions
716 // to use the same register (e.g., due to overlapping access to low/high
717 // bits of the same packed register), then only the *second* (latter)
718 // instruction should mark the register as killed. This is because the
719 // second instruction handles the higher bits and is effectively the last
720 // user of the full register pair.
721
722 bool OpSel = SrcMods & SISrcMods::OP_SEL_0;
723 bool OpSelHi = SrcMods & SISrcMods::OP_SEL_1;
724 bool KillState = true;
725 if ((OpSel == OpSelHi) && !IsHiBits)
726 KillState = false;
727 UnpackedSrcMO.setIsKill(KillState);
728 }
729 NewMI.add(MO: UnpackedSrcMO);
730}
731
732void SIPreEmitPeephole::collectUnpackingCandidates(
733 MachineInstr &BeginMI, SetVector<MachineInstr *> &InstrsToUnpack,
734 uint16_t NumMFMACycles) {
735 auto *BB = BeginMI.getParent();
736 auto E = BB->end();
737 int TotalCyclesBetweenCandidates = 0;
738 auto SchedModel = TII->getSchedModel();
739 Register MFMADef = BeginMI.getOperand(i: 0).getReg();
740
741 for (auto I = std::next(x: BeginMI.getIterator()); I != E; ++I) {
742 MachineInstr &Instr = *I;
743 uint32_t UnpackedOpCode = mapToUnpackedOpcode(I&: Instr);
744 bool IsUnpackable =
745 !(UnpackedOpCode == std::numeric_limits<uint32_t>::max());
746 if (Instr.isMetaInstruction())
747 continue;
748 if ((Instr.isTerminator()) ||
749 (TII->isNeverCoissue(MI&: Instr) && !IsUnpackable) ||
750 (SIInstrInfo::modifiesModeRegister(MI: Instr) &&
751 Instr.modifiesRegister(Reg: AMDGPU::EXEC, TRI)))
752 return;
753
754 const MCSchedClassDesc *InstrSchedClassDesc =
755 SchedModel.resolveSchedClass(MI: &Instr);
756 uint16_t Latency =
757 SchedModel.getWriteProcResBegin(SC: InstrSchedClassDesc)->ReleaseAtCycle;
758 TotalCyclesBetweenCandidates += Latency;
759
760 if (TotalCyclesBetweenCandidates >= NumMFMACycles - 1)
761 return;
762 // Identify register dependencies between those used by the MFMA
763 // instruction and the following packed instructions. Also checks for
764 // transitive dependencies between the MFMA def and candidate instruction
765 // def and uses. Conservatively ensures that we do not incorrectly
766 // read/write registers.
767 for (const MachineOperand &InstrMO : Instr.operands()) {
768 if (!InstrMO.isReg() || !InstrMO.getReg().isValid())
769 continue;
770 if (TRI->regsOverlap(RegA: MFMADef, RegB: InstrMO.getReg()))
771 return;
772 }
773 if (!IsUnpackable)
774 continue;
775
776 if (canUnpackingClobberRegister(MI: Instr))
777 return;
778 // If it's a packed instruction, adjust latency: remove the packed
779 // latency, add latency of two unpacked instructions (currently estimated
780 // as 2 cycles).
781 TotalCyclesBetweenCandidates -= Latency;
782 // TODO: improve latency handling based on instruction modeling.
783 TotalCyclesBetweenCandidates += 2;
784 // Subtract 1 to account for MFMA issue latency.
785 if (TotalCyclesBetweenCandidates < NumMFMACycles - 1)
786 InstrsToUnpack.insert(X: &Instr);
787 }
788}
789
790void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
791 const MachineOperand &DstOp = I.getOperand(i: 0);
792
793 uint32_t UnpackedOpcode = mapToUnpackedOpcode(I);
794 assert(UnpackedOpcode != std::numeric_limits<uint32_t>::max() &&
795 "Unsupported Opcode");
796
797 MachineInstrBuilder Op0LOp1L =
798 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
799 MachineOperand LoDstOp = Op0LOp1L->getOperand(i: 0);
800
801 LoDstOp.setIsUndef(DstOp.isUndef());
802
803 MachineInstrBuilder Op0HOp1H =
804 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
805 MachineOperand HiDstOp = Op0HOp1H->getOperand(i: 0);
806
807 uint32_t IFlags = I.getFlags();
808 Op0LOp1L->setFlags(IFlags);
809 Op0HOp1H->setFlags(IFlags);
810 LoDstOp.setIsRenamable(DstOp.isRenamable());
811 HiDstOp.setIsRenamable(DstOp.isRenamable());
812
813 I.eraseFromParent();
814}
815
816MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
817 uint32_t UnpackedOpcode,
818 bool IsHiBits) {
819 MachineBasicBlock &MBB = *I.getParent();
820 const DebugLoc &DL = I.getDebugLoc();
821 const MachineOperand *SrcMO0 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0);
822 const MachineOperand *SrcMO1 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1);
823 Register DstReg = I.getOperand(i: 0).getReg();
824 unsigned OpCode = I.getOpcode();
825 Register UnpackedDstReg = IsHiBits ? TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub1)
826 : TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
827
828 int64_t ClampVal = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::clamp)->getImm();
829 unsigned Src0Mods =
830 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
831 unsigned Src1Mods =
832 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
833
834 MachineInstrBuilder NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: UnpackedOpcode));
835 NewMI.addDef(RegNo: UnpackedDstReg); // vdst
836 addOperandAndMods(NewMI, SrcMods: Src0Mods, IsHiBits, SrcMO: *SrcMO0);
837 addOperandAndMods(NewMI, SrcMods: Src1Mods, IsHiBits, SrcMO: *SrcMO1);
838
839 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
840 const MachineOperand *SrcMO2 =
841 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2);
842 unsigned Src2Mods =
843 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
844 addOperandAndMods(NewMI, SrcMods: Src2Mods, IsHiBits, SrcMO: *SrcMO2);
845 }
846 NewMI.addImm(Val: ClampVal); // clamp
847 // Packed instructions do not support output modifiers. safe to assign them 0
848 // for this use case
849 NewMI.addImm(Val: 0); // omod
850 return NewMI;
851}
852
853PreservedAnalyses
854llvm::SIPreEmitPeepholePass::run(MachineFunction &MF,
855 MachineFunctionAnalysisManager &MFAM) {
856 auto *MLI = MFAM.getCachedResult<MachineLoopAnalysis>(IR&: MF);
857 SIPreEmitPeephole Impl;
858
859 if (Impl.run(MF, MLI)) {
860 auto PA = getMachineFunctionPassPreservedAnalyses();
861 PA.preserve<MachineLoopAnalysis>();
862 return PA;
863 }
864
865 return PreservedAnalyses::all();
866}
867
868bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
869 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
870 TII = ST.getInstrInfo();
871 TRI = &TII->getRegisterInfo();
872 MLI = LoopInfo;
873 bool Changed = false;
874
875 MF.RenumberBlocks();
876
877 for (MachineBasicBlock &MBB : MF) {
878 Changed |= removeRedundantModeWrites(SrcMBB&: MBB);
879
880 MachineBasicBlock::iterator TermI = MBB.getFirstTerminator();
881 // Check first terminator for branches to optimize
882 if (TermI != MBB.end()) {
883 MachineInstr &MI = *TermI;
884 switch (MI.getOpcode()) {
885 case AMDGPU::S_CBRANCH_VCCZ:
886 case AMDGPU::S_CBRANCH_VCCNZ:
887 Changed |= optimizeVccBranch(MI);
888 break;
889 case AMDGPU::S_CBRANCH_EXECZ:
890 Changed |= removeExeczBranch(MI, SrcMBB&: MBB);
891 break;
892 }
893 }
894
895 if (!ST.hasVGPRIndexMode())
896 continue;
897
898 MachineInstr *SetGPRMI = nullptr;
899 const unsigned Threshold = 20;
900 unsigned Count = 0;
901 // Scan the block for two S_SET_GPR_IDX_ON instructions to see if a
902 // second is not needed. Do expensive checks in the optimizeSetGPR()
903 // and limit the distance to 20 instructions for compile time purposes.
904 // Note: this needs to work on bundles as S_SET_GPR_IDX* instructions
905 // may be bundled with the instructions they modify.
906 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
907 if (Count == Threshold)
908 SetGPRMI = nullptr;
909 else
910 ++Count;
911
912 if (MI.getOpcode() != AMDGPU::S_SET_GPR_IDX_ON)
913 continue;
914
915 Count = 0;
916 if (!SetGPRMI) {
917 SetGPRMI = &MI;
918 continue;
919 }
920
921 if (optimizeSetGPR(First&: *SetGPRMI, MI))
922 Changed = true;
923 else
924 SetGPRMI = &MI;
925 }
926 }
927
928 // TODO: Fold this into previous block, if possible. Evaluate and handle any
929 // side effects.
930
931 // Perform the extra MF scans only for supported archs
932 if (!ST.hasGFX940Insts())
933 return Changed;
934 for (MachineBasicBlock &MBB : MF) {
935 // Unpack packed instructions overlapped by MFMAs. This allows the
936 // compiler to co-issue unpacked instructions with MFMA
937 auto SchedModel = TII->getSchedModel();
938 SetVector<MachineInstr *> InstrsToUnpack;
939 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
940 if (!SIInstrInfo::isMFMA(MI))
941 continue;
942 const MCSchedClassDesc *SchedClassDesc =
943 SchedModel.resolveSchedClass(MI: &MI);
944 uint16_t NumMFMACycles =
945 SchedModel.getWriteProcResBegin(SC: SchedClassDesc)->ReleaseAtCycle;
946 collectUnpackingCandidates(BeginMI&: MI, InstrsToUnpack, NumMFMACycles);
947 }
948 for (MachineInstr *MI : InstrsToUnpack) {
949 performF32Unpacking(I&: *MI);
950 }
951 }
952
953 return Changed;
954}
955